diff --git a/build/pom.xml b/build/pom.xml index 5a097bb382a..57d54f39046 100755 --- a/build/pom.xml +++ b/build/pom.xml @@ -631,6 +631,11 @@ hostNetworkInterface ${project.version} + + org.zstack + physicalServer + ${project.version} + org.zstack ovn @@ -977,6 +982,11 @@ hostNetworkInterface ${project.version} + + org.zstack + physicalServer + ${project.version} + org.zstack observabilityServer diff --git a/conf/db/upgrade/V5.5.38__schema.sql b/conf/db/upgrade/V5.5.38__schema.sql index 1b8937fe5e9..c666c0051a4 100644 --- a/conf/db/upgrade/V5.5.38__schema.sql +++ b/conf/db/upgrade/V5.5.38__schema.sql @@ -20,3 +20,88 @@ CREATE TABLE IF NOT EXISTS `zstack`.`SnmpEngineVO` ( `lastOpDate` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`uuid`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`PhysicalServerVO` ( + `uuid` varchar(32) NOT NULL, + `zoneUuid` varchar(32) DEFAULT NULL, + `serialNumber` varchar(255) NOT NULL, + `createDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + `lastOpDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00' ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`uuid`), + UNIQUE KEY `ukPhysicalServerSerialNumber` (`serialNumber`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`PhysicalServerResourceAssignmentVO` ( + `uuid` varchar(32) NOT NULL, + `serverUuid` varchar(32) NOT NULL, + `roleType` varchar(64) NOT NULL, + `cpuSet` varchar(4096) NOT NULL DEFAULT '', + `memory` bigint unsigned DEFAULT NULL, + `state` varchar(32) NOT NULL, + `createDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + `lastOpDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00' ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`uuid`), + UNIQUE KEY `ukPhysicalServerResourceAssignment` (`serverUuid`, `roleType`), + CONSTRAINT `fkPhysicalServerResourceAssignmentServerUuid` + FOREIGN KEY (`serverUuid`) REFERENCES `zstack`.`PhysicalServerVO` (`uuid`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CALL ADD_COLUMN('HostEO', 'serverUuid', 'VARCHAR(32)', 1, NULL); +CALL ADD_COLUMN('ManagementNodeVO', 'serverUuid', 'VARCHAR(32)', 1, NULL); + +UPDATE `zstack`.`HostEO` +SET `serverUuid` = NULL +WHERE `deleted` IS NOT NULL AND `serverUuid` IS NOT NULL; + +DROP PROCEDURE IF EXISTS addPhysicalServerIdentityUniqueKeys; +DELIMITER $$ +CREATE PROCEDURE addPhysicalServerIdentityUniqueKeys() +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = 'zstack' + AND table_name = 'HostEO' + AND index_name = 'ukHostEOServerUuid' + ) THEN + ALTER TABLE `zstack`.`HostEO` + ADD UNIQUE KEY `ukHostEOServerUuid` (`serverUuid`); + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = 'zstack' + AND table_name = 'ManagementNodeVO' + AND index_name = 'ukManagementNodeVOServerUuid' + ) THEN + ALTER TABLE `zstack`.`ManagementNodeVO` + ADD UNIQUE KEY `ukManagementNodeVOServerUuid` (`serverUuid`); + END IF; +END $$ +DELIMITER ; +CALL addPhysicalServerIdentityUniqueKeys(); +DROP PROCEDURE IF EXISTS addPhysicalServerIdentityUniqueKeys; + +CALL ADD_CONSTRAINT( + 'HostEO', + 'fkHostEOServerUuid', + 'serverUuid', + 'PhysicalServerVO', + 'uuid', + 'SET NULL' +); + +DROP VIEW IF EXISTS `zstack`.`HostVO`; +CREATE VIEW `zstack`.`HostVO` AS + SELECT uuid, zoneUuid, clusterUuid, name, description, managementIp, hypervisorType, + state, status, createDate, lastOpDate, architecture, serverUuid + FROM `zstack`.`HostEO` + WHERE deleted IS NULL; + +CALL ADD_CONSTRAINT( + 'ManagementNodeVO', + 'fkManagementNodeVOServerUuid', + 'serverUuid', + 'PhysicalServerVO', + 'uuid', + 'SET NULL' +); diff --git a/conf/persistence.xml b/conf/persistence.xml index e76141af3ad..371ebd523cc 100755 --- a/conf/persistence.xml +++ b/conf/persistence.xml @@ -229,5 +229,7 @@ org.zstack.network.hostNetworkInterface.PhysicalSwitchVO org.zstack.network.hostNetworkInterface.PhysicalSwitchPortVO org.zstack.header.core.external.service.ExternalServiceConfigurationVO + org.zstack.physicalserver.PhysicalServerVO + org.zstack.physicalserver.PhysicalServerResourceAssignmentVO diff --git a/conf/serviceConfig/physicalServer.xml b/conf/serviceConfig/physicalServer.xml new file mode 100644 index 00000000000..f72bf1e9f39 --- /dev/null +++ b/conf/serviceConfig/physicalServer.xml @@ -0,0 +1,27 @@ + + + physicalServer + PhysicalServerApiInterceptor + + + org.zstack.physicalserver.APIQueryPhysicalServerMsg + query + + + + org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentMsg + query + + + + org.zstack.physicalserver.APIGetPhysicalServerManagedServicesMsg + + + + org.zstack.physicalserver.APIUpdatePhysicalServerResourceAssignmentMsg + + + + org.zstack.physicalserver.APIRefreshPhysicalServerResourceAssignmentsMsg + + diff --git a/conf/springConfigXml/HostManager.xml b/conf/springConfigXml/HostManager.xml index 8147620c4ae..67605682a22 100755 --- a/conf/springConfigXml/HostManager.xml +++ b/conf/springConfigXml/HostManager.xml @@ -74,4 +74,4 @@ - + diff --git a/conf/springConfigXml/Kvm.xml b/conf/springConfigXml/Kvm.xml index a96965fc01b..d1124099a69 100755 --- a/conf/springConfigXml/Kvm.xml +++ b/conf/springConfigXml/Kvm.xml @@ -115,6 +115,24 @@ + + + + + + + + + + + + + + + + + diff --git a/conf/springConfigXml/ManagementNodeManager.xml b/conf/springConfigXml/ManagementNodeManager.xml index ab2d7eefaa1..f861397dd7a 100755 --- a/conf/springConfigXml/ManagementNodeManager.xml +++ b/conf/springConfigXml/ManagementNodeManager.xml @@ -17,4 +17,18 @@ + + + + + + + + + + + + + diff --git a/conf/springConfigXml/physicalServer.xml b/conf/springConfigXml/physicalServer.xml new file mode 100644 index 00000000000..06cc6225caa --- /dev/null +++ b/conf/springConfigXml/physicalServer.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/springConfigXml/zbs.xml b/conf/springConfigXml/zbs.xml index 2e0234ddc1d..329c01bb33c 100644 --- a/conf/springConfigXml/zbs.xml +++ b/conf/springConfigXml/zbs.xml @@ -29,4 +29,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/zstack.xml b/conf/zstack.xml index 17bbdfae090..a194805937e 100755 --- a/conf/zstack.xml +++ b/conf/zstack.xml @@ -46,6 +46,7 @@ + diff --git a/core/src/main/java/org/zstack/core/Platform.java b/core/src/main/java/org/zstack/core/Platform.java index 34df20e3e12..9bb9913dbf7 100755 --- a/core/src/main/java/org/zstack/core/Platform.java +++ b/core/src/main/java/org/zstack/core/Platform.java @@ -120,8 +120,10 @@ public class Platform { }; private static final List UNUSABLE_MACHINE_IDENTITIES = Arrays.asList( "none", + "n/a", "unknown", "not specified", + "to be filled", "to be filled by o.e.m.", "to be filled by oem", "default string", @@ -896,6 +898,18 @@ private static String getManagementServerFingerprint() { return MANAGEMENT_SERVER_FINGERPRINT_ALGORITHM + DigestUtils.sha256Hex(StringUtils.join(identities, "\n")); } + public static String getManagementServerSerialNumber() { + String serialNumber = normalizeMachineSerialNumber( + readMachineIdentity("/sys/class/dmi/id/product_serial")); + return serialNumber == null + ? normalizeMachineSerialNumber(readDmiMachineIdentity("system-serial-number")) + : serialNumber; + } + + public static String normalizeMachineSerialNumber(String serialNumber) { + return normalizeMachineIdentity(serialNumber); + } + private static String readMachineIdentity(String path) { File file = new File(path); if (!file.isFile() || !file.canRead()) { diff --git a/header/src/main/java/org/zstack/header/host/HostAO.java b/header/src/main/java/org/zstack/header/host/HostAO.java index 8f7bd58ef95..52c2f11a6f6 100755 --- a/header/src/main/java/org/zstack/header/host/HostAO.java +++ b/header/src/main/java/org/zstack/header/host/HostAO.java @@ -37,6 +37,9 @@ public class HostAO extends ResourceVO { @Column private String architecture; + @Column + private String serverUuid; + @Column @Enumerated(EnumType.STRING) private HostState state; @@ -146,4 +149,12 @@ public String getArchitecture() { public void setArchitecture(String architecture) { this.architecture = architecture; } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } } diff --git a/header/src/main/java/org/zstack/header/host/HostAO_.java b/header/src/main/java/org/zstack/header/host/HostAO_.java index 921e2b5ac64..c6568126895 100755 --- a/header/src/main/java/org/zstack/header/host/HostAO_.java +++ b/header/src/main/java/org/zstack/header/host/HostAO_.java @@ -21,4 +21,5 @@ public class HostAO_ extends ResourceVO_ { public static volatile SingularAttribute createDate; public static volatile SingularAttribute lastOpDate; public static volatile SingularAttribute architecture; + public static volatile SingularAttribute serverUuid; } diff --git a/header/src/main/java/org/zstack/header/host/HostInventory.java b/header/src/main/java/org/zstack/header/host/HostInventory.java index 35958b05003..38d6baf8f93 100755 --- a/header/src/main/java/org/zstack/header/host/HostInventory.java +++ b/header/src/main/java/org/zstack/header/host/HostInventory.java @@ -185,6 +185,8 @@ public class HostInventory implements Serializable { private String architecture; + private String serverUuid; + /** * @desc the time this resource gets created */ @@ -207,6 +209,7 @@ protected HostInventory(HostVO vo) { this.setZoneUuid(vo.getZoneUuid()); this.setClusterUuid(vo.getClusterUuid()); this.setArchitecture(vo.getArchitecture()); + this.setServerUuid(vo.getServerUuid()); if (vo.getCapacity() != null) { this.setTotalCpuCapacity(vo.getCapacity().getTotalCpu()); this.setAvailableCpuCapacity(vo.getCapacity().getAvailableCpu()); @@ -395,6 +398,14 @@ public void setArchitecture(String architecture) { this.architecture = architecture; } + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + public String getIpmiAddress() { return ipmiAddress; } diff --git a/header/src/main/java/org/zstack/header/host/HostNUMANode.java b/header/src/main/java/org/zstack/header/host/HostNUMANode.java index 733a90cc828..39993485e36 100644 --- a/header/src/main/java/org/zstack/header/host/HostNUMANode.java +++ b/header/src/main/java/org/zstack/header/host/HostNUMANode.java @@ -7,6 +7,8 @@ public class HostNUMANode { public List distance; public List cpus; + public List onlineCpus; + public List> coreGroups; public Long free; public Long size; public String nodeID; @@ -36,6 +38,22 @@ public void setCpus(List cpus) { this.cpus = cpus; } + public List getOnlineCpus() { + return onlineCpus; + } + + public void setOnlineCpus(List onlineCpus) { + this.onlineCpus = onlineCpus; + } + + public List> getCoreGroups() { + return coreGroups; + } + + public void setCoreGroups(List> coreGroups) { + this.coreGroups = coreGroups; + } + public Long getFree() { return free; } diff --git a/header/src/main/java/org/zstack/header/host/HostVO.java b/header/src/main/java/org/zstack/header/host/HostVO.java index 319c26750c4..48f0067e435 100755 --- a/header/src/main/java/org/zstack/header/host/HostVO.java +++ b/header/src/main/java/org/zstack/header/host/HostVO.java @@ -80,6 +80,6 @@ protected HostVO(HostVO vo) { this.setCapacity(vo.getCapacity()); this.setIpmi(vo.getIpmi()); this.setHwMonitorStatus(vo.getHwMonitorStatus()); + this.setServerUuid(vo.getServerUuid()); } } - diff --git a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java index 97ce31aec22..f19d8a74c8b 100755 --- a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java +++ b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java @@ -17,6 +17,7 @@ public class ManagementNodeInventory { private String hostName; private Timestamp joinDate; private Timestamp heartBeat; + private String serverUuid; public static ManagementNodeInventory valueOf(ManagementNodeVO vo) { ManagementNodeInventory inv = new ManagementNodeInventory(); @@ -24,6 +25,7 @@ public static ManagementNodeInventory valueOf(ManagementNodeVO vo) { inv.setHostName(vo.getHostName()); inv.setJoinDate(vo.getJoinDate()); inv.setUuid(vo.getUuid()); + inv.setServerUuid(vo.getServerUuid()); return inv; } @@ -66,4 +68,12 @@ public Timestamp getJoinDate() { public Timestamp getHeartBeat() { return heartBeat; } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } } diff --git a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java index bd89bcbbf3d..d82cb6ec0e9 100755 --- a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java +++ b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java @@ -26,6 +26,9 @@ public class ManagementNodeVO { @Column private int port; + @Column + private String serverUuid; + public String getHostName() { return hostName; } @@ -73,4 +76,12 @@ public void setJoinDate(Timestamp joinDate) { public void setHeartBeat(Timestamp heartBeat) { this.heartBeat = heartBeat; } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } } diff --git a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java index 70035f974cc..81d98c0450f 100755 --- a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java +++ b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java @@ -13,4 +13,5 @@ public class ManagementNodeVO_ { public static volatile SingularAttribute joinDate; public static volatile SingularAttribute heartBeat; public static volatile SingularAttribute state; + public static volatile SingularAttribute serverUuid; } diff --git a/header/src/main/java/org/zstack/header/physicalserver/ManagedServiceResourceUsage.java b/header/src/main/java/org/zstack/header/physicalserver/ManagedServiceResourceUsage.java new file mode 100644 index 00000000000..8519574014a --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/ManagedServiceResourceUsage.java @@ -0,0 +1,76 @@ +package org.zstack.header.physicalserver; + +public class ManagedServiceResourceUsage { + private String roleType; + private String serviceName; + private boolean restartable; + private String state; + private String cpuSet; + private Long cpuTime; + private Long memory; + private Long memoryLimit; + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public boolean isRestartable() { + return restartable; + } + + public void setRestartable(boolean restartable) { + this.restartable = restartable; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getCpuTime() { + return cpuTime; + } + + public void setCpuTime(Long cpuTime) { + this.cpuTime = cpuTime; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public Long getMemoryLimit() { + return memoryLimit; + } + + public void setMemoryLimit(Long memoryLimit) { + this.memoryLimit = memoryLimit; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuSet.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuSet.java new file mode 100644 index 00000000000..7358acb290d --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuSet.java @@ -0,0 +1,199 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.List; +import java.math.BigInteger; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +public final class PhysicalServerCpuSet { + private PhysicalServerCpuSet() { + } + + public static String normalize(String value) { + return formatRanges(parseRanges(value)); + } + + public static SortedSet parse(String value, Set allowed) { + List ranges = parseRanges(value); + SortedSet cpus = new TreeSet<>(); + for (Range range : ranges) { + long width = (long) range.end - range.start + 1; + if (width > allowed.size()) { + throw new IllegalArgumentException(String.format( + "CPU_SET_INVALID: CPU_SET range[%s-%s] contains CPUs outside the online topology", + range.start, range.end)); + } + for (long cursor = range.start; cursor <= range.end; cursor++) { + int cpu = (int) cursor; + if (!allowed.contains(cpu)) { + throw new IllegalArgumentException(String.format( + "CPU_SET_INVALID: CPU[%s] is outside the online topology", cpu)); + } + cpus.add(cpu); + } + } + return cpus; + } + + public static SortedSet parse(String value) { + List ranges = parseRanges(value); + SortedSet cpus = new TreeSet<>(); + for (Range range : ranges) { + long width = (long) range.end - range.start + 1; + if (width > 1048576) { + throw new IllegalArgumentException(String.format( + "CPU_SET_INVALID: CPU_SET range[%s-%s] is too large", + range.start, range.end)); + } + for (long cursor = range.start; cursor <= range.end; cursor++) { + cpus.add((int) cursor); + } + } + return cpus; + } + + public static String format(Collection cpus) { + if (cpus == null || cpus.isEmpty()) { + return ""; + } + List sorted = new ArrayList<>(new LinkedHashSet<>(cpus)); + Collections.sort(sorted); + List ranges = new ArrayList<>(); + int start = sorted.get(0); + int end = start; + for (int index = 1; index < sorted.size(); index++) { + int current = sorted.get(index); + if ((long) current == (long) end + 1) { + end = current; + } else { + ranges.add(new Range(start, end)); + start = current; + end = current; + } + } + ranges.add(new Range(start, end)); + return formatRanges(ranges); + } + + public static int count(String value) { + if (value == null || value.trim().isEmpty()) { + return 0; + } + long count = 0; + for (Range range : parseRanges(value)) { + count += (long) range.end - range.start + 1; + if (count > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + "CPU_SET_INVALID: CPU_SET contains too many CPUs"); + } + } + return (int) count; + } + + public static String union(String left, String right) { + boolean leftEmpty = left == null || left.trim().isEmpty(); + boolean rightEmpty = right == null || right.trim().isEmpty(); + if (leftEmpty && rightEmpty) { + return ""; + } + if (leftEmpty) { + return normalize(right); + } + if (rightEmpty) { + return normalize(left); + } + return normalize(left + "," + right); + } + + private static List parseRanges(String value) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException("CPU_SET_INVALID: CPU_SET cannot be empty"); + } + + List ranges = new ArrayList<>(); + for (String item : value.split(",")) { + String part = item.trim(); + if (part.isEmpty()) { + throw new IllegalArgumentException( + "CPU_SET_INVALID: CPU_SET contains an empty item"); + } + + String[] range = part.split("-", -1); + if (range.length > 2) { + throw new IllegalArgumentException(String.format( + "CPU_SET_INVALID: invalid CPU_SET item[%s]", part)); + } + + int start = parseCpu(range[0]); + int end = range.length == 1 ? start : parseCpu(range[1]); + if (start > end) { + throw new IllegalArgumentException(String.format( + "CPU_SET_INVALID: invalid CPU_SET range[%s]", part)); + } + ranges.add(new Range(start, end)); + } + + ranges.sort(Comparator.comparingInt(range -> range.start)); + List mergedRanges = new ArrayList<>(); + Range merged = ranges.get(0); + for (int index = 1; index < ranges.size(); index++) { + Range current = ranges.get(index); + if ((long) current.start <= (long) merged.end + 1) { + merged.end = Math.max(merged.end, current.end); + } else { + mergedRanges.add(merged); + merged = current; + } + } + mergedRanges.add(merged); + return mergedRanges; + } + + private static String formatRanges(List ranges) { + StringBuilder result = new StringBuilder(); + for (Range range : ranges) { + appendRange(result, range.start, range.end); + } + return result.toString(); + } + + private static int parseCpu(String value) { + String normalized = value.trim(); + if (!normalized.matches("[0-9]+")) { + throw new IllegalArgumentException(String.format( + "CPU_SET_INVALID: invalid CPU id[%s]", value)); + } + BigInteger cpu = new BigInteger(normalized); + if (cpu.compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0) { + throw new IllegalArgumentException(String.format( + "CPU_SET_INVALID: CPU id[%s] is too large", value)); + } + return cpu.intValue(); + } + + private static void appendRange(StringBuilder result, int start, int end) { + if (result.length() > 0) { + result.append(','); + } + result.append(start); + if (start != end) { + result.append('-').append(end); + } + } + + private static class Range { + private final int start; + private int end; + + private Range(int start, int end) { + this.start = start; + this.end = end; + } + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuTopology.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuTopology.java new file mode 100644 index 00000000000..41dfbcbedf4 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuTopology.java @@ -0,0 +1,129 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +public class PhysicalServerCpuTopology { + public static class CoreGroup { + private final String numaId; + private final SortedSet cpus; + + CoreGroup(String numaId, Collection cpus) { + this.numaId = numaId; + this.cpus = new TreeSet<>(cpus); + } + + public String getNumaId() { + return numaId; + } + + public SortedSet getCpus() { + return new TreeSet<>(cpus); + } + + int firstCpu() { + return cpus.first(); + } + } + + private final SortedSet onlineCpus; + private final List coreGroups; + + private PhysicalServerCpuTopology(SortedSet onlineCpus, List coreGroups) { + this.onlineCpus = onlineCpus; + this.coreGroups = coreGroups; + } + + public static PhysicalServerCpuTopology from(Map numaNodes) { + if (numaNodes == null || numaNodes.isEmpty()) { + throw new IllegalArgumentException("CPU_TOPOLOGY_UNAVAILABLE: NUMA topology is empty"); + } + + SortedSet online = new TreeSet<>(); + List groups = new ArrayList<>(); + Set grouped = new HashSet<>(); + Map sortedNodes = new LinkedHashMap<>(); + numaNodes.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> sortedNodes.put(entry.getKey(), entry.getValue())); + + for (Map.Entry entry : sortedNodes.entrySet()) { + PhysicalServerNumaNode node = entry.getValue(); + if (node.getOnlineCpus() == null || node.getCoreGroups() == null || node.getCoreGroups().isEmpty()) { + throw new IllegalArgumentException( + "CPU_TOPOLOGY_UNSUPPORTED: topology provider did not report online CPUs and core sibling groups"); + } + + Set nodeOnline = parseCpuIds(node.getOnlineCpus()); + online.addAll(nodeOnline); + for (List rawGroup : node.getCoreGroups()) { + Set group = parseCpuIds(rawGroup); + if (group.isEmpty() || !nodeOnline.containsAll(group)) { + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: core group is outside its NUMA online CPUs"); + } + for (Integer cpu : group) { + if (!grouped.add(cpu)) { + throw new IllegalArgumentException(String.format( + "CPU_TOPOLOGY_INVALID: CPU[%s] appears in multiple core groups", cpu)); + } + } + groups.add(new CoreGroup(entry.getKey(), group)); + } + } + + if (!grouped.equals(online)) { + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: core groups do not cover every online CPU exactly once"); + } + groups.sort(Comparator.comparingInt(CoreGroup::firstCpu)); + return new PhysicalServerCpuTopology(online, groups); + } + + private static Set parseCpuIds(List values) { + Set result = new HashSet<>(); + for (String value : values) { + if (value == null || !value.matches("[0-9]+")) { + throw new IllegalArgumentException(String.format("CPU_TOPOLOGY_INVALID: invalid CPU id[%s]", value)); + } + BigInteger cpu = new BigInteger(value); + if (cpu.compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0) { + throw new IllegalArgumentException(String.format("CPU_TOPOLOGY_INVALID: CPU id[%s] is too large", value)); + } + result.add(cpu.intValue()); + } + return result; + } + + public SortedSet getOnlineCpus() { + return new TreeSet<>(onlineCpus); + } + + public List getCoreGroups() { + return new ArrayList<>(coreGroups); + } + + public CoreGroup getCpuZeroGroup() { + for (CoreGroup group : coreGroups) { + if (group.cpus.contains(0)) { + return group; + } + } + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: online topology does not contain CPU0"); + } + + public String fingerprint() { + List encoded = new ArrayList<>(); + for (CoreGroup group : coreGroups) { + encoded.add(group.numaId + ":" + PhysicalServerCpuSet.format(group.cpus)); + } + return String.join(";", encoded); + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerIdentitySpec.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerIdentitySpec.java new file mode 100644 index 00000000000..0c9c6111ee2 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerIdentitySpec.java @@ -0,0 +1,30 @@ +package org.zstack.header.physicalserver; + +public class PhysicalServerIdentitySpec { + private String serialNumber; + private String zoneUuid; + + public PhysicalServerIdentitySpec() { + } + + public PhysicalServerIdentitySpec(String serialNumber, String zoneUuid) { + this.serialNumber = serialNumber; + this.zoneUuid = zoneUuid; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public String getZoneUuid() { + return zoneUuid; + } + + public void setZoneUuid(String zoneUuid) { + this.zoneUuid = zoneUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerManager.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerManager.java new file mode 100644 index 00000000000..723d3b580f1 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerManager.java @@ -0,0 +1,40 @@ +package org.zstack.header.physicalserver; + +import org.zstack.header.core.Completion; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +public interface PhysicalServerManager { + Map resolveIdentities(Collection identities); + + Map findSerialNumbersByServerUuids( + Collection serverUuids); + + void ensureResourceAssignments(Collection serverUuids, String roleType); + + default void ensureResourceAssignment(String serverUuid, String roleType) { + ensureResourceAssignments(Collections.singleton(serverUuid), roleType); + } + + void reconcile(String serverUuid, boolean refreshFacts); + + void reconcileAll(); + + default void releaseResourceAssignment( + String serverUuid, + String roleType, + String consumerUuid, + Completion completion) { + releaseResourceAssignment( + serverUuid, roleType, consumerUuid, false, completion); + } + + void releaseResourceAssignment( + String serverUuid, + String roleType, + String consumerUuid, + boolean force, + Completion completion); +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerNumaNode.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerNumaNode.java new file mode 100644 index 00000000000..01725e9b0e6 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerNumaNode.java @@ -0,0 +1,34 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.List; + +public class PhysicalServerNumaNode { + private String nodeId; + private List onlineCpus = new ArrayList<>(); + private List> coreGroups = new ArrayList<>(); + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public List getOnlineCpus() { + return onlineCpus; + } + + public void setOnlineCpus(List onlineCpus) { + this.onlineCpus = onlineCpus; + } + + public List> getCoreGroups() { + return coreGroups; + } + + public void setCoreGroups(List> coreGroups) { + this.coreGroups = coreGroups; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceApplicationMode.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceApplicationMode.java new file mode 100644 index 00000000000..be58aca894f --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceApplicationMode.java @@ -0,0 +1,6 @@ +package org.zstack.header.physicalserver; + +public enum PhysicalServerResourceApplicationMode { + RESOURCE_HANDLES, + PROVIDER_MANAGED +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceConsumerState.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceConsumerState.java new file mode 100644 index 00000000000..ed212af9f8d --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceConsumerState.java @@ -0,0 +1,7 @@ +package org.zstack.header.physicalserver; + +public enum PhysicalServerResourceConsumerState { + AVAILABLE, + UNAVAILABLE, + MISSING +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceControlAdapter.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceControlAdapter.java new file mode 100644 index 00000000000..c23dc66e7e3 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceControlAdapter.java @@ -0,0 +1,76 @@ +package org.zstack.header.physicalserver; + +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +public interface PhysicalServerResourceControlAdapter { + String getRoleType(); + + PhysicalServerResourceIsolationMode getIsolationMode(); + + PhysicalServerResourceApplicationMode getApplicationMode(); + + default String getTopologyRoleType() { + return getRoleType(); + } + + default String getDefaultCpuSet( + PhysicalServerCpuTopology topology, + Set allocatedExclusiveCpus) { + Set available = new TreeSet<>(topology.getOnlineCpus()); + available.removeAll(allocatedExclusiveCpus); + return PhysicalServerCpuSet.format(available); + } + + default void refreshAssociations() { + } + + default void refreshAssociations(Collection serverUuids) { + refreshAssociations(); + } + + default void refreshCapacity(String serverUuid) { + } + + Set getAssociatedServerUuids(); + + default Set getEligibleDefaultServerUuids() { + return getAssociatedServerUuids(); + } + + PhysicalServerResourceConsumerState getState(String serverUuid); + + default String getUnavailableReason(String serverUuid) { + return "RESOURCE_CONSUMER_UNAVAILABLE"; + } + + Map getStates( + Collection serverUuids); + + void collectTopology( + String serverUuid, + ReturnValueCompletion completion); + + void apply( + String serverUuid, + String consumerUuid, + ResourceControlCommand command, + ReturnValueCompletion completion); + + void collectManagedServiceUsage( + String serverUuid, + boolean includeAuxiliaryServices, + ReturnValueCompletion> completion); + + void restartManagedServices( + String serverUuid, + boolean includeAuxiliaryServices, + Collection serviceNames, + Completion completion); +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceIsolationMode.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceIsolationMode.java new file mode 100644 index 00000000000..a0265b9aaa3 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerResourceIsolationMode.java @@ -0,0 +1,6 @@ +package org.zstack.header.physicalserver; + +public enum PhysicalServerResourceIsolationMode { + SHARED, + EXCLUSIVE +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/ResourceConsumerHandle.java b/header/src/main/java/org/zstack/header/physicalserver/ResourceConsumerHandle.java new file mode 100644 index 00000000000..67c8f220016 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/ResourceConsumerHandle.java @@ -0,0 +1,90 @@ +package org.zstack.header.physicalserver; + +public class ResourceConsumerHandle { + public static final String SYSTEMD_UNIT = "SYSTEMD_UNIT"; + public static final String OWNER_PID_FILE = "OWNER_PID_FILE"; + + private String handleType; + private String value; + private String serviceName; + private String consumerKey; + private boolean optional; + private boolean restartable; + private String expectedCommandToken; + + public ResourceConsumerHandle() { + } + + public ResourceConsumerHandle( + String handleType, + String value, + String serviceName, + String consumerKey, + boolean optional, + boolean restartable, + String expectedCommandToken) { + this.handleType = handleType; + this.value = value; + this.serviceName = serviceName; + this.consumerKey = consumerKey; + this.optional = optional; + this.restartable = restartable; + this.expectedCommandToken = expectedCommandToken; + } + + public String getHandleType() { + return handleType; + } + + public void setHandleType(String handleType) { + this.handleType = handleType; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public String getConsumerKey() { + return consumerKey; + } + + public void setConsumerKey(String consumerKey) { + this.consumerKey = consumerKey; + } + + public boolean isOptional() { + return optional; + } + + public void setOptional(boolean optional) { + this.optional = optional; + } + + public boolean isRestartable() { + return restartable; + } + + public void setRestartable(boolean restartable) { + this.restartable = restartable; + } + + public String getExpectedCommandToken() { + return expectedCommandToken; + } + + public void setExpectedCommandToken(String expectedCommandToken) { + this.expectedCommandToken = expectedCommandToken; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/ResourceControlCommand.java b/header/src/main/java/org/zstack/header/physicalserver/ResourceControlCommand.java new file mode 100644 index 00000000000..89b203bc793 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/ResourceControlCommand.java @@ -0,0 +1,79 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.List; + +public class ResourceControlCommand { + private String roleType; + private String isolationMode; + private String operation; + private String cpuSet; + private Long memory; + private String sliceName; + private boolean includeAuxiliaryServices; + private List handles = new ArrayList<>(); + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getIsolationMode() { + return isolationMode; + } + + public void setIsolationMode(String isolationMode) { + this.isolationMode = isolationMode; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public String getSliceName() { + return sliceName; + } + + public void setSliceName(String sliceName) { + this.sliceName = sliceName; + } + + public boolean isIncludeAuxiliaryServices() { + return includeAuxiliaryServices; + } + + public void setIncludeAuxiliaryServices(boolean includeAuxiliaryServices) { + this.includeAuxiliaryServices = includeAuxiliaryServices; + } + + public List getHandles() { + return handles; + } + + public void setHandles(List handles) { + this.handles = handles; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/ResourceControlResponse.java b/header/src/main/java/org/zstack/header/physicalserver/ResourceControlResponse.java new file mode 100644 index 00000000000..1df5cdf009c --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/ResourceControlResponse.java @@ -0,0 +1,61 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.List; + +public class ResourceControlResponse { + private String state; + private String cpuSet; + private Long memory; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private List results = new ArrayList<>(); + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public List getResults() { + return results; + } + + public void setResults(List results) { + this.results = results; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/ResourceControlResult.java b/header/src/main/java/org/zstack/header/physicalserver/ResourceControlResult.java new file mode 100644 index 00000000000..aa939f3bde3 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/ResourceControlResult.java @@ -0,0 +1,31 @@ +package org.zstack.header.physicalserver; + +public class ResourceControlResult { + private String state; + private String cpuSet; + private Long memory; + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/RoleServiceManifest.java b/header/src/main/java/org/zstack/header/physicalserver/RoleServiceManifest.java new file mode 100644 index 00000000000..4a08e1240e3 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/RoleServiceManifest.java @@ -0,0 +1,375 @@ +package org.zstack.header.physicalserver; + +import org.zstack.utils.YamlUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class RoleServiceManifest { + private String roleType; + private String applicationMode; + private String sliceName; + private List services = new ArrayList<>(); + + public static RoleServiceManifest load( + String resourcePath, + String expectedRoleType, + PhysicalServerResourceApplicationMode expectedApplicationMode) { + RoleServiceManifest manifest = YamlUtils.load( + read(resourcePath), RoleServiceManifest.class); + if (manifest == null) { + throw new IllegalStateException(String.format( + "role service manifest[%s] is empty", resourcePath)); + } + manifest.validate( + resourcePath, expectedRoleType, expectedApplicationMode); + manifest.services = Collections.unmodifiableList( + new ArrayList<>(manifest.services)); + return manifest; + } + + public List handles( + String primaryConsumerKey, + String auxiliaryConsumerKey, + boolean includeAuxiliaryServices) { + return handles( + primaryConsumerKey, + auxiliaryConsumerKey, + includeAuxiliaryServices, + Collections.emptyMap()); + } + + public List handles( + String primaryConsumerKey, + String auxiliaryConsumerKey, + boolean includeAuxiliaryServices, + Map values) { + if (!PhysicalServerResourceApplicationMode.RESOURCE_HANDLES.name() + .equals(applicationMode)) { + throw new IllegalStateException(String.format( + "role[%s] does not use resource handles", roleType)); + } + List result = new ArrayList<>(); + for (Service service : services) { + if (service.isAuxiliary() && !includeAuxiliaryServices) { + continue; + } + result.add(new ResourceConsumerHandle( + service.getHandleType(), + service.resolveValue(values), + service.getName(), + service.isAuxiliary() + ? auxiliaryConsumerKey : primaryConsumerKey, + !service.required(), + service.restartable(), + service.getExpectedCommandToken())); + } + return result; + } + + public List handlesByServiceNames( + Collection serviceNames, + String primaryConsumerKey, + String auxiliaryConsumerKey, + boolean includeAuxiliaryServices, + Map values) { + if (serviceNames == null || serviceNames.isEmpty()) { + throw new IllegalArgumentException("service names must not be empty"); + } + Set selected = new HashSet<>(serviceNames); + if (selected.size() != serviceNames.size()) { + throw new IllegalArgumentException("service names must not be duplicated"); + } + List available = handles( + primaryConsumerKey, auxiliaryConsumerKey, + includeAuxiliaryServices, values); + List result = available.stream() + .filter(handle -> selected.remove(handle.getServiceName())) + .collect(Collectors.toList()); + if (!selected.isEmpty()) { + throw new IllegalArgumentException(String.format( + "services%s are not defined by role[%s]", selected, roleType)); + } + return result; + } + + public List managedServiceUsages( + boolean includeAuxiliaryServices, String state) { + List result = new ArrayList<>(); + for (Service service : services) { + if (service.isAuxiliary() && !includeAuxiliaryServices) { + continue; + } + ManagedServiceResourceUsage usage = + new ManagedServiceResourceUsage(); + usage.setRoleType(roleType); + usage.setServiceName(service.getName()); + usage.setRestartable(service.restartable()); + usage.setState(state); + result.add(usage); + } + return result; + } + + private void validate( + String resourcePath, + String expectedRoleType, + PhysicalServerResourceApplicationMode expectedApplicationMode) { + if (!expectedRoleType.equals(roleType)) { + throw invalid(resourcePath, String.format( + "roleType[%s] does not match expected roleType[%s]", + roleType, expectedRoleType)); + } + if (!expectedApplicationMode.name().equals(applicationMode)) { + throw invalid(resourcePath, String.format( + "applicationMode[%s] does not match expected mode[%s]", + applicationMode, expectedApplicationMode)); + } + if (expectedApplicationMode + == PhysicalServerResourceApplicationMode.RESOURCE_HANDLES + && (empty(sliceName) + || !sliceName.matches("[A-Za-z0-9][A-Za-z0-9_.@:-]{0,248}\\.slice"))) { + throw invalid(resourcePath, String.format( + "sliceName[%s] is invalid", sliceName)); + } + if (expectedApplicationMode + == PhysicalServerResourceApplicationMode.PROVIDER_MANAGED + && !empty(sliceName)) { + throw invalid(resourcePath, + "provider-managed role cannot define sliceName"); + } + if (services == null || services.isEmpty()) { + throw invalid(resourcePath, "services must not be empty"); + } + Set names = new HashSet<>(); + for (Service service : services) { + if (service == null || empty(service.getName()) + || !service.getName().matches( + "[A-Za-z0-9][A-Za-z0-9_.-]{0,63}")) { + throw invalid(resourcePath, "service name must not be empty"); + } + if (!names.add(service.getName())) { + throw invalid(resourcePath, String.format( + "service name[%s] is duplicated", service.getName())); + } + if (expectedApplicationMode + == PhysicalServerResourceApplicationMode.RESOURCE_HANDLES) { + validateHandle(resourcePath, service); + } + } + } + + private void validateHandle(String resourcePath, Service service) { + if (!ResourceConsumerHandle.SYSTEMD_UNIT.equals(service.getHandleType()) + && !ResourceConsumerHandle.OWNER_PID_FILE.equals( + service.getHandleType())) { + throw invalid(resourcePath, String.format( + "service[%s] has unsupported handleType[%s]", + service.getName(), service.getHandleType())); + } + if (empty(service.getValue()) == empty(service.getValueFrom())) { + throw invalid(resourcePath, String.format( + "service[%s] must specify exactly one of value and valueFrom", + service.getName())); + } + if (service.getRequired() == null) { + throw invalid(resourcePath, String.format( + "service[%s] required must be specified", service.getName())); + } + if (service.getRestartable() == null) { + throw invalid(resourcePath, String.format( + "service[%s] restartable must be specified", + service.getName())); + } + if (service.restartable() + && !ResourceConsumerHandle.SYSTEMD_UNIT.equals( + service.getHandleType())) { + throw invalid(resourcePath, String.format( + "service[%s] restartable requires a systemd handle", + service.getName())); + } + if (ResourceConsumerHandle.OWNER_PID_FILE.equals( + service.getHandleType()) + && empty(service.getExpectedCommandToken())) { + throw invalid(resourcePath, String.format( + "service[%s] command token must not be empty", + service.getName())); + } + if (ResourceConsumerHandle.SYSTEMD_UNIT.equals(service.getHandleType()) + && !empty(service.getExpectedCommandToken())) { + throw invalid(resourcePath, String.format( + "service[%s] systemd handle cannot have a command token", + service.getName())); + } + } + + private static IllegalStateException invalid( + String resourcePath, String reason) { + return new IllegalStateException(String.format( + "invalid role service manifest[%s]: %s", resourcePath, reason)); + } + + private static boolean empty(String value) { + return value == null || value.trim().isEmpty(); + } + + private static String read(String resourcePath) { + ClassLoader context = Thread.currentThread().getContextClassLoader(); + InputStream stream = context == null + ? null : context.getResourceAsStream(resourcePath); + if (stream == null) { + stream = RoleServiceManifest.class.getClassLoader() + .getResourceAsStream(resourcePath); + } + if (stream == null) { + throw new IllegalStateException(String.format( + "role service manifest[%s] was not found", resourcePath)); + } + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + stream, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } catch (IOException error) { + throw new IllegalStateException(String.format( + "cannot read role service manifest[%s]", resourcePath), error); + } + } + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getApplicationMode() { + return applicationMode; + } + + public void setApplicationMode(String applicationMode) { + this.applicationMode = applicationMode; + } + + public String getSliceName() { + return sliceName; + } + + public void setSliceName(String sliceName) { + this.sliceName = sliceName; + } + + public List getServices() { + return services; + } + + public void setServices(List services) { + this.services = services; + } + + public static class Service { + private String name; + private String handleType; + private String value; + private String valueFrom; + private Boolean required; + private Boolean restartable; + private boolean auxiliary; + private String expectedCommandToken; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getHandleType() { + return handleType; + } + + public void setHandleType(String handleType) { + this.handleType = handleType; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getValueFrom() { + return valueFrom; + } + + public void setValueFrom(String valueFrom) { + this.valueFrom = valueFrom; + } + + private String resolveValue(Map values) { + if (!empty(value)) { + return value; + } + String resolved = values.get(valueFrom); + if (empty(resolved)) { + throw new IllegalStateException(String.format( + "role service[%s] valueFrom[%s] cannot be resolved", + name, valueFrom)); + } + return resolved; + } + + public Boolean getRequired() { + return required; + } + + public void setRequired(Boolean required) { + this.required = required; + } + + private boolean required() { + return Boolean.TRUE.equals(required); + } + + public Boolean getRestartable() { + return restartable; + } + + public void setRestartable(Boolean restartable) { + this.restartable = restartable; + } + + private boolean restartable() { + return Boolean.TRUE.equals(restartable); + } + + public boolean isAuxiliary() { + return auxiliary; + } + + public void setAuxiliary(boolean auxiliary) { + this.auxiliary = auxiliary; + } + + public String getExpectedCommandToken() { + return expectedCommandToken; + } + + public void setExpectedCommandToken(String expectedCommandToken) { + this.expectedCommandToken = expectedCommandToken; + } + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java index feb4c5c11c7..daedf838732 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java @@ -43,6 +43,10 @@ public interface PrimaryStorageControllerSvc { */ void syncConfig(String config); + default void beforePersistAddonInfo(AddonInfo addonInfo, Completion completion) { + completion.success(); + } + void getCapacity(List requiredUrls, ReturnValueCompletion comp); void reportCapacity(ReturnValueCompletion comp); void reportHealthy(ReturnValueCompletion comp); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java index 16ff667f21d..de9e012e022 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java @@ -30,10 +30,14 @@ public class KVMHostCapacityExtension implements KVMHostConnectExtensionPoint, H private ResourceConfigFacade rcf; public void reportCapacity(HostInventory host, Completion completion) { + reportCapacity(host.getUuid(), completion); + } + + public void reportCapacity(String hostUuid, Completion completion) { CheckHostCapacityMsg msg = new CheckHostCapacityMsg(); - msg.setHostUuid(host.getUuid()); + msg.setHostUuid(hostUuid); msg.setTimeout(TimeUnit.MINUTES.toMillis(30)); - bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, host.getUuid()); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); bus.send(msg, new CloudBusCallBack(completion) { @Override public void run(MessageReply rly) { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerAdapter.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerAdapter.java new file mode 100644 index 00000000000..53e729dc960 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerAdapter.java @@ -0,0 +1,988 @@ +package org.zstack.kvm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.compute.host.HostSystemTags; +import org.zstack.compute.host.PostHostConnectExtensionPoint; +import org.zstack.core.Platform; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.Component; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorableValue; +import org.zstack.header.host.GetHostNumaTopologyMsg; +import org.zstack.header.host.GetHostNumaTopologyReply; +import org.zstack.header.host.HostAO_; +import org.zstack.header.host.HostAfterConnectedExtensionPoint; +import org.zstack.header.host.HostConstant; +import org.zstack.header.host.HostDeleteExtensionPoint; +import org.zstack.header.host.HostEO; +import org.zstack.header.host.HostException; +import org.zstack.header.host.HostInventory; +import org.zstack.header.host.HostNUMANode; +import org.zstack.header.host.HostStatus; +import org.zstack.header.host.HostVO; +import org.zstack.header.host.HostVO_; +import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter; +import org.zstack.header.physicalserver.PhysicalServerResourceApplicationMode; +import org.zstack.header.physicalserver.PhysicalServerResourceConsumerState; +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; +import org.zstack.header.physicalserver.PhysicalServerNumaNode; +import org.zstack.header.physicalserver.ResourceControlCommand; +import org.zstack.header.physicalserver.ResourceControlResponse; +import org.zstack.header.physicalserver.ResourceConsumerHandle; +import org.zstack.header.physicalserver.ResourceControlResult; +import org.zstack.header.physicalserver.RoleServiceManifest; +import org.zstack.header.tag.AbstractSystemTagLifeCycleListener; +import org.zstack.header.tag.SystemTagInventory; +import org.zstack.header.tag.SystemTagVO; +import org.zstack.header.tag.SystemTagVO_; +import org.zstack.utils.TagUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import javax.persistence.Tuple; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.zstack.core.Platform.operr; + +public class KvmPhysicalServerAdapter implements + PhysicalServerResourceControlAdapter, + PostHostConnectExtensionPoint, + HostAfterConnectedExtensionPoint, + HostDeleteExtensionPoint, + ManagementNodeReadyExtensionPoint, + Component { + public static final String ROLE_TYPE = "COMPUTE"; + public static final String APPLY_RESOURCE_CONTROL_PATH = "/host/resourcecontrol/apply"; + public static final String GET_MANAGED_SERVICE_USAGE_PATH = + "/host/resourcecontrol/services"; + public static final String RESTART_MANAGED_SERVICES_PATH = + "/host/resourcecontrol/restart"; + public static final String ROLE_SERVICE_MANIFEST_PATH = + "physical-server-roles/compute.yaml"; + private static final String ERROR_CODE = "ORG_ZSTACK_KVM_10000"; + private static final int HOST_LINK_UPDATE_BATCH_SIZE = 500; + private static final CLogger logger = Utils.getLogger(KvmPhysicalServerAdapter.class); + private static final RoleServiceManifest ROLE_SERVICES = + RoleServiceManifest.load( + ROLE_SERVICE_MANIFEST_PATH, + ROLE_TYPE, + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES); + private final AtomicReference> hostRelations = + new AtomicReference<>(Collections.emptyMap()); + + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + @Autowired + private DatabaseFacade dbf; + @Autowired + private CloudBus bus; + @Autowired + private KVMHostCapacityExtension capacityExtension; + + @Override + public String getRoleType() { + return ROLE_TYPE; + } + + @Override + public PhysicalServerResourceIsolationMode getIsolationMode() { + return PhysicalServerResourceIsolationMode.SHARED; + } + + @Override + public PhysicalServerResourceApplicationMode getApplicationMode() { + return PhysicalServerResourceApplicationMode.RESOURCE_HANDLES; + } + + @Override + public Set getAssociatedServerUuids() { + if (hostRelations.get().isEmpty()) { + refreshHostRelations(Collections.emptySet()); + } + return new LinkedHashSet<>(hostRelations.get().keySet()); + } + + @Override + public void refreshAssociations() { + backfill(false); + refreshHostRelations(Collections.emptySet()); + } + + @Override + public void refreshAssociations(Collection serverUuids) { + if (serverUuids == null || serverUuids.isEmpty()) { + refreshAssociations(); + return; + } + refreshHostRelations(serverUuids); + } + + @Override + public PhysicalServerResourceConsumerState getState(String serverUuid) { + HostRelation host = hostRelation(serverUuid); + if (host == null) { + return PhysicalServerResourceConsumerState.MISSING; + } + return host.status == HostStatus.Connected + ? PhysicalServerResourceConsumerState.AVAILABLE + : PhysicalServerResourceConsumerState.UNAVAILABLE; + } + + @Override + public Map getStates( + Collection serverUuids) { + if (serverUuids == null || serverUuids.isEmpty()) { + return Collections.emptyMap(); + } + refreshMissingHostRelations(serverUuids); + Map result = new HashMap<>(); + for (String serverUuid : serverUuids) { + HostRelation relation = hostRelations.get().get(serverUuid); + result.put(serverUuid, relation == null + ? PhysicalServerResourceConsumerState.MISSING + : relation.status == HostStatus.Connected + ? PhysicalServerResourceConsumerState.AVAILABLE + : PhysicalServerResourceConsumerState.UNAVAILABLE); + } + return result; + } + + @Override + public void refreshCapacity(String serverUuid) { + String hostUuid = hostUuid(serverUuid, null); + if (hostUuid == null) { + return; + } + capacityExtension.reportCapacity(hostUuid, new Completion(null) { + @Override + public void success() { + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to refresh capacity for host[uuid:%s] after " + + "physical server resource assignment changed: %s", + hostUuid, errorCode)); + } + }); + } + + @Override + public void collectTopology( + String serverUuid, + ReturnValueCompletion completion) { + String hostUuid = hostUuid(serverUuid, null); + if (hostUuid == null) { + completion.fail(operr(ERROR_CODE, + "HOST_RELATION_MISSING: physical server[uuid:%s] has no KVM host", + serverUuid)); + return; + } + + GetHostNumaTopologyMsg msg = new GetHostNumaTopologyMsg(); + msg.setHostUuid(hostUuid); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + completeTopology(reply.castReply(), completion); + } + }); + } + + private void completeTopology( + GetHostNumaTopologyReply reply, + ReturnValueCompletion completion) { + try { + completion.success(PhysicalServerCpuTopology.from( + neutralTopology(reply.getNuma()))); + } catch (RuntimeException error) { + completion.fail(operr(ERROR_CODE, + "CPU_TOPOLOGY_FACT_INVALID: %s", error.getMessage())); + } + } + + @Override + public void apply( + String serverUuid, + String consumerUuid, + ResourceControlCommand command, + ReturnValueCompletion completion) { + String hostUuid = hostUuid(serverUuid, consumerUuid); + if (hostUuid == null) { + completion.fail(operr(ERROR_CODE, + "HOST_RELATION_MISSING: physical server[uuid:%s] has no matching KVM host", + serverUuid)); + return; + } + + ResourceControlAgentCommand agentCommand = new ResourceControlAgentCommand(); + agentCommand.setRoleType(command.getRoleType()); + agentCommand.setOperation(command.getOperation()); + agentCommand.setCpuSet(command.getCpuSet()); + agentCommand.setMemory(command.getMemory()); + agentCommand.setSliceName(ROLE_SERVICES.getSliceName()); + agentCommand.setHandles(handles(hostUuid, command.isIncludeAuxiliaryServices())); + + KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); + msg.setPath(APPLY_RESOURCE_CONTROL_PATH); + msg.setHostUuid(hostUuid); + msg.setCommand(agentCommand); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + ErrorableValue result = + KVMHostAsyncHttpCallReply.unwrap( + reply, ResourceControlAgentResponse.class); + if (!result.isSuccess()) { + completion.fail(result.error); + return; + } + completion.success(result.result.toInventory()); + } + }); + } + + @Override + public void collectManagedServiceUsage( + String serverUuid, + boolean includeAuxiliaryServices, + ReturnValueCompletion> completion) { + String hostUuid = hostUuid(serverUuid, null); + if (hostUuid == null) { + completion.success(ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE")); + return; + } + ManagedServiceAgentCommand command = new ManagedServiceAgentCommand(); + command.setRoleType(ROLE_TYPE); + command.setSliceName(ROLE_SERVICES.getSliceName()); + command.setHandles(handles(hostUuid, includeAuxiliaryServices)); + KVMHostAsyncHttpCallMsg msg = managedServiceCall( + hostUuid, GET_MANAGED_SERVICE_USAGE_PATH, command); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + ErrorableValue result = + KVMHostAsyncHttpCallReply.unwrap( + reply, ManagedServiceUsageAgentResponse.class); + if (!result.isSuccess()) { + logger.warn(String.format( + "failed to query managed services on host[uuid:%s]: %s", + hostUuid, result.error)); + completion.success(ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE")); + return; + } + List services = + result.result.getServices(); + if (services == null) { + completion.success(ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE")); + return; + } + for (ManagedServiceResourceUsage usage : services) { + usage.setRoleType(ROLE_TYPE); + } + completion.success(services); + } + }); + } + + @Override + public void restartManagedServices( + String serverUuid, + boolean includeAuxiliaryServices, + Collection serviceNames, + Completion completion) { + String hostUuid = hostUuid(serverUuid, null); + if (hostUuid == null) { + completion.fail(operr(ERROR_CODE, + "HOST_RELATION_MISSING: physical server[uuid:%s] has no KVM host", + serverUuid)); + return; + } + ManagedServiceAgentCommand command = new ManagedServiceAgentCommand(); + command.setRoleType(ROLE_TYPE); + command.setSliceName(ROLE_SERVICES.getSliceName()); + try { + command.setHandles(ROLE_SERVICES.handlesByServiceNames( + serviceNames, + String.format("host-agent:%s", hostUuid), + String.format("host-agent:%s", hostUuid), + includeAuxiliaryServices, + Collections.emptyMap())); + } catch (RuntimeException error) { + completion.fail(operr(ERROR_CODE, "%s", error.getMessage())); + return; + } + KVMHostAsyncHttpCallMsg msg = managedServiceCall( + hostUuid, RESTART_MANAGED_SERVICES_PATH, command); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + ErrorableValue result = + KVMHostAsyncHttpCallReply.unwrap( + reply, KVMAgentCommands.AgentResponse.class); + if (!result.isSuccess()) { + completion.fail(result.error); + return; + } + completion.success(); + } + }); + } + + private KVMHostAsyncHttpCallMsg managedServiceCall( + String hostUuid, String path, ManagedServiceAgentCommand command) { + KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); + msg.setPath(path); + msg.setHostUuid(hostUuid); + msg.setCommand(command); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + return msg; + } + + @Override + public Flow createPostHostConnectFlow(HostInventory host) { + return new NoRollbackFlow() { + String __name__ = "associate-kvm-host-with-physical-server"; + + @Override + public void run(FlowTrigger trigger, Map data) { + associate(host); + trigger.next(); + } + }; + } + + @Override + public void afterHostConnected(HostInventory host) { + if (physicalServerManager == null) { + return; + } + String serverUuid = Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .eq(HostVO_.uuid, host.getUuid()) + .findValue(); + if (serverUuid != null) { + physicalServerManager.reconcile(serverUuid, true); + } + } + + @Override + public void preDeleteHost(HostInventory inventory) throws HostException { + } + + @Override + public void beforeDeleteHost(HostInventory inventory) { + if (inventory.getServerUuid() == null) { + return; + } + String serverUuid = inventory.getServerUuid(); + org.zstack.core.db.SQL.New(HostEO.class) + .eq(HostAO_.uuid, inventory.getUuid()) + .eq(HostAO_.serverUuid, serverUuid) + .set(HostAO_.serverUuid, null) + .update(); + removeHostRelation(serverUuid); + } + + @Override + public void afterDeleteHost(HostInventory inventory) { + if (inventory.getServerUuid() == null || physicalServerManager == null) { + return; + } + removeHostRelation(inventory.getServerUuid()); + physicalServerManager.reconcile(inventory.getServerUuid(), true); + } + + @Override + public void managementNodeReady() { + backfill(true); + } + + @Override + public boolean start() { + HostSystemTags.SYSTEM_SERIAL_NUMBER.installLifeCycleListener( + new AbstractSystemTagLifeCycleListener() { + @Override + public void tagCreated(SystemTagInventory tag) { + backfill(Collections.singleton(tag.getResourceUuid()), true); + } + + @Override + public void tagUpdated( + SystemTagInventory old, SystemTagInventory newTag) { + backfill(Collections.singleton(newTag.getResourceUuid()), true); + } + }); + return true; + } + + @Override + public boolean stop() { + return true; + } + + @Transactional + public void associate(HostInventory host) { + if (physicalServerManager == null + || !KVMConstant.KVM_HYPERVISOR_TYPE.equals(host.getHypervisorType())) { + return; + } + String current = Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .eq(HostVO_.uuid, host.getUuid()) + .findValue(); + if (current != null) { + physicalServerManager.reconcile(current, true); + return; + } + + String serialNumber = Platform.normalizeMachineSerialNumber( + HostSystemTags.SYSTEM_SERIAL_NUMBER.getTokenByResourceUuid( + host.getUuid(), HostSystemTags.SYSTEM_SERIAL_NUMBER_TOKEN)); + if (serialNumber == null) { + logger.warn(String.format( + "cannot associate host[uuid:%s] with a physical server, " + + "reasonCode[IDENTITY_UNAVAILABLE]", host.getUuid())); + return; + } + String zoneUuid = host.getZoneUuid() == null + ? Q.New(HostVO.class).select(HostVO_.zoneUuid) + .eq(HostVO_.uuid, host.getUuid()).findValue() + : host.getZoneUuid(); + String serverUuid = physicalServerManager.resolveIdentities( + Collections.singletonList( + new PhysicalServerIdentitySpec(serialNumber, zoneUuid))) + .get(serialNumber); + if (serverUuid == null) { + return; + } + clearDeletedHostLink(serverUuid); + if (linkedHost(serverUuid, host.getUuid()) != null) { + return; + } + + Query update = dbf.getEntityManager().createNativeQuery( + "UPDATE IGNORE HostEO SET serverUuid = :serverUuid " + + "WHERE uuid = :hostUuid AND serverUuid IS NULL"); + update.setParameter("serverUuid", serverUuid); + update.setParameter("hostUuid", host.getUuid()); + update.executeUpdate(); + current = Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .eq(HostVO_.uuid, host.getUuid()) + .findValue(); + if (!serverUuid.equals(current)) { + logger.warn(String.format( + "cannot associate host[uuid:%s] with physical server[uuid:%s], " + + "reasonCode[IDENTITY_ASSOCIATION_CONFLICT]", + host.getUuid(), serverUuid)); + return; + } + physicalServerManager.reconcile(serverUuid, true); + } + + @Transactional + private void backfill(boolean requestReconcile) { + backfill(Collections.emptySet(), requestReconcile); + } + + @Transactional + private void backfill( + Collection hostUuids, boolean requestReconcile) { + if (physicalServerManager == null) { + return; + } + Q hostQuery = Q.New(HostVO.class) + .select(HostVO_.uuid, HostVO_.zoneUuid) + .eq(HostVO_.hypervisorType, KVMConstant.KVM_HYPERVISOR_TYPE) + .isNull(HostVO_.serverUuid); + if (hostUuids != null && !hostUuids.isEmpty()) { + hostQuery.in(HostVO_.uuid, hostUuids); + } + List hostTuples = hostQuery.listTuple(); + if (hostTuples.isEmpty()) { + return; + } + Map zonesByHost = new LinkedHashMap<>(); + for (Tuple tuple : hostTuples) { + zonesByHost.put(tuple.get(0, String.class), tuple.get(1, String.class)); + } + Map> serialsByHost = serialsByHost(zonesByHost.keySet()); + Map> candidatesBySerial = new LinkedHashMap<>(); + for (Map.Entry host : zonesByHost.entrySet()) { + Set serials = serialsByHost.get(host.getKey()); + if (serials == null || serials.size() != 1) { + continue; + } + candidatesBySerial + .computeIfAbsent(serials.iterator().next(), ignored -> new ArrayList<>()) + .add(new HostCandidate(host.getKey(), host.getValue())); + } + + Map candidates = new LinkedHashMap<>(); + List identities = new ArrayList<>(); + for (Map.Entry> candidate : candidatesBySerial.entrySet()) { + if (candidate.getValue().size() != 1) { + logger.warn(String.format( + "cannot backfill host physical server association for serialNumber[%s], " + + "reasonCode[IDENTITY_ALREADY_ASSOCIATED]", + candidate.getKey())); + continue; + } + HostCandidate host = candidate.getValue().get(0); + candidates.put(candidate.getKey(), host); + identities.add(new PhysicalServerIdentitySpec(candidate.getKey(), host.zoneUuid)); + } + Map resolved = physicalServerManager.resolveIdentities(identities); + clearDeletedHostLinks(resolved.values()); + Set used = new HashSet<>(Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .notNull(HostVO_.serverUuid) + .listValues()); + Map links = new LinkedHashMap<>(); + for (Map.Entry candidate : candidates.entrySet()) { + String serverUuid = resolved.get(candidate.getKey()); + if (serverUuid != null && used.add(serverUuid)) { + links.put(candidate.getValue().hostUuid, serverUuid); + } + } + updateHostLinks(links); + + List linked = links.isEmpty() ? Collections.emptyList() : Q.New(HostVO.class) + .select(HostVO_.uuid, HostVO_.serverUuid) + .in(HostVO_.uuid, links.keySet()) + .notNull(HostVO_.serverUuid) + .listTuple(); + Set linkedServers = new LinkedHashSet<>(); + for (Tuple tuple : linked) { + String serverUuid = tuple.get(1, String.class); + linkedServers.add(serverUuid); + } + if (requestReconcile && !linkedServers.isEmpty()) { + for (String serverUuid : linkedServers) { + physicalServerManager.reconcile(serverUuid, true); + } + } + } + + private Map> serialsByHost(Collection hostUuids) { + List tagTuples = Q.New(SystemTagVO.class) + .select(SystemTagVO_.resourceUuid, SystemTagVO_.tag) + .in(SystemTagVO_.resourceUuid, hostUuids) + .eq(SystemTagVO_.resourceType, HostVO.class.getSimpleName()) + .like(SystemTagVO_.tag, TagUtils.tagPatternToSqlPattern( + HostSystemTags.SYSTEM_SERIAL_NUMBER.getTagFormat())) + .listTuple(); + Map> result = new HashMap<>(); + for (Tuple tuple : tagTuples) { + String hostUuid = tuple.get(0, String.class); + String serialNumber = Platform.normalizeMachineSerialNumber( + HostSystemTags.SYSTEM_SERIAL_NUMBER.getTokenByTag( + tuple.get(1, String.class), + HostSystemTags.SYSTEM_SERIAL_NUMBER_TOKEN)); + if (serialNumber != null) { + result.computeIfAbsent(hostUuid, ignored -> new LinkedHashSet<>()) + .add(serialNumber); + } + } + return result; + } + + private void updateHostLinks(Map links) { + if (links.isEmpty()) { + return; + } + List> entries = + new ArrayList<>(links.entrySet()); + for (int start = 0; start < entries.size(); + start += HOST_LINK_UPDATE_BATCH_SIZE) { + updateHostLinks(entries.subList( + start, + Math.min(start + HOST_LINK_UPDATE_BATCH_SIZE, entries.size()))); + } + } + + private void updateHostLinks(List> links) { + StringBuilder sql = new StringBuilder( + "UPDATE IGNORE HostEO SET serverUuid = CASE uuid"); + int index = 0; + for (Map.Entry ignored : links) { + sql.append(" WHEN :host").append(index) + .append(" THEN :server").append(index); + index++; + } + sql.append(" ELSE serverUuid END WHERE serverUuid IS NULL AND uuid IN ("); + for (int i = 0; i < links.size(); i++) { + if (i > 0) { + sql.append(','); + } + sql.append(":host").append(i); + } + sql.append(')'); + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + index = 0; + for (Map.Entry link : links) { + query.setParameter("host" + index, link.getKey()); + query.setParameter("server" + index, link.getValue()); + index++; + } + query.executeUpdate(); + } + + private void clearDeletedHostLinks(Collection serverUuids) { + Set targets = new LinkedHashSet<>(serverUuids); + targets.remove(null); + if (targets.isEmpty()) { + return; + } + StringBuilder sql = new StringBuilder( + "UPDATE HostEO SET serverUuid = NULL " + + "WHERE deleted IS NOT NULL AND serverUuid IN ("); + for (int i = 0; i < targets.size(); i++) { + if (i > 0) { + sql.append(','); + } + sql.append(":server").append(i); + } + sql.append(')'); + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + int index = 0; + for (String serverUuid : targets) { + query.setParameter("server" + index, serverUuid); + index++; + } + query.executeUpdate(); + } + + private void clearDeletedHostLink(String serverUuid) { + Query query = dbf.getEntityManager().createNativeQuery( + "UPDATE HostEO SET serverUuid = NULL " + + "WHERE deleted IS NOT NULL AND serverUuid = :serverUuid"); + query.setParameter("serverUuid", serverUuid); + query.executeUpdate(); + } + + private String linkedHost(String serverUuid, String excludedHostUuid) { + return Q.New(HostVO.class) + .select(HostVO_.uuid) + .eq(HostVO_.serverUuid, serverUuid) + .notEq(HostVO_.uuid, excludedHostUuid) + .findValue(); + } + + private String hostUuid(String serverUuid, String consumerUuid) { + HostRelation relation = hostRelation(serverUuid); + if (relation == null) { + return null; + } + return consumerUuid == null || consumerUuid.equals(relation.hostUuid) + ? relation.hostUuid : null; + } + + private HostRelation hostRelation(String serverUuid) { + HostRelation relation = hostRelations.get().get(serverUuid); + if (relation != null) { + return relation; + } + refreshHostRelations(Collections.singleton(serverUuid)); + return hostRelations.get().get(serverUuid); + } + + private void refreshMissingHostRelations(Collection serverUuids) { + Set missing = new LinkedHashSet<>(serverUuids); + missing.removeAll(hostRelations.get().keySet()); + if (!missing.isEmpty()) { + refreshHostRelations(missing); + } + } + + private void refreshHostRelations(Collection serverUuids) { + Q query = Q.New(HostVO.class) + .select(HostVO_.serverUuid, HostVO_.uuid, HostVO_.status) + .eq(HostVO_.hypervisorType, KVMConstant.KVM_HYPERVISOR_TYPE) + .notNull(HostVO_.serverUuid); + boolean partial = serverUuids != null && !serverUuids.isEmpty(); + if (partial) { + query.in(HostVO_.serverUuid, serverUuids); + } + Map loaded = new HashMap<>(); + for (Tuple host : (List) query.listTuple()) { + loaded.put( + host.get(0, String.class), + new HostRelation( + host.get(1, String.class), + host.get(2, HostStatus.class))); + } + if (!partial) { + hostRelations.set(Collections.unmodifiableMap(loaded)); + return; + } + while (true) { + Map current = hostRelations.get(); + Map replacement = new HashMap<>(current); + for (String serverUuid : serverUuids) { + replacement.remove(serverUuid); + } + replacement.putAll(loaded); + if (hostRelations.compareAndSet( + current, Collections.unmodifiableMap(replacement))) { + return; + } + } + } + + private void removeHostRelation(String serverUuid) { + while (true) { + Map current = hostRelations.get(); + if (!current.containsKey(serverUuid)) { + return; + } + Map replacement = new HashMap<>(current); + replacement.remove(serverUuid); + if (hostRelations.compareAndSet( + current, Collections.unmodifiableMap(replacement))) { + return; + } + } + } + + private static class HostRelation { + private final String hostUuid; + private final HostStatus status; + + private HostRelation(String hostUuid, HostStatus status) { + this.hostUuid = hostUuid; + this.status = status; + } + } + + private Map neutralTopology( + Map topology) { + Map result = new LinkedHashMap<>(); + if (topology == null) { + return result; + } + for (Map.Entry entry : topology.entrySet()) { + PhysicalServerNumaNode node = new PhysicalServerNumaNode(); + node.setNodeId(entry.getKey()); + node.setOnlineCpus(entry.getValue().getOnlineCpus()); + node.setCoreGroups(entry.getValue().getCoreGroups()); + result.put(entry.getKey(), node); + } + return result; + } + + private List handles( + String hostUuid, boolean includeAuxiliaryServices) { + String consumerKey = String.format("host-agent:%s", hostUuid); + return ROLE_SERVICES.handles( + consumerKey, consumerKey, includeAuxiliaryServices); + } + + public static class ResourceControlAgentCommand extends KVMAgentCommands.AgentCommand { + private String roleType; + private String operation; + private String cpuSet; + private Long memory; + private String sliceName; + private List handles = new ArrayList<>(); + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public String getSliceName() { + return sliceName; + } + + public void setSliceName(String sliceName) { + this.sliceName = sliceName; + } + + public List getHandles() { + return handles; + } + + public void setHandles(List handles) { + this.handles = handles; + } + } + + public static class ManagedServiceAgentCommand extends KVMAgentCommands.AgentCommand { + private String roleType; + private String sliceName; + private List handles = new ArrayList<>(); + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getSliceName() { + return sliceName; + } + + public void setSliceName(String sliceName) { + this.sliceName = sliceName; + } + + public List getHandles() { + return handles; + } + + public void setHandles(List handles) { + this.handles = handles; + } + } + + public static class ManagedServiceUsageAgentResponse extends KVMAgentCommands.AgentResponse { + private List services = new ArrayList<>(); + + public List getServices() { + return services; + } + + public void setServices(List services) { + this.services = services; + } + } + + public static class ResourceControlAgentResponse extends KVMAgentCommands.AgentResponse { + private String cpuSet; + private Long memory; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private List results = new ArrayList<>(); + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public List getResults() { + return results; + } + + public void setResults(List results) { + this.results = results; + } + + private ResourceControlResponse toInventory() { + ResourceControlResponse response = new ResourceControlResponse(); + response.setCpuSet(cpuSet); + response.setMemory(memory); + response.setCoveredServiceCount(coveredServiceCount); + response.setExpectedServiceCount(expectedServiceCount); + response.setResults(results); + return response; + } + } + + private static class HostCandidate { + private final String hostUuid; + private final String zoneUuid; + + private HostCandidate(String hostUuid, String zoneUuid) { + this.hostUuid = hostUuid; + this.zoneUuid = zoneUuid; + } + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerResourceCascadeExtension.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerResourceCascadeExtension.java new file mode 100644 index 00000000000..beb745dedcb --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerResourceCascadeExtension.java @@ -0,0 +1,113 @@ +package org.zstack.kvm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cascade.AbstractAsyncCascadeExtension; +import org.zstack.core.cascade.CascadeAction; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.host.HostInventory; +import org.zstack.header.host.HostVO; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class KvmPhysicalServerResourceCascadeExtension extends AbstractAsyncCascadeExtension { + private static final CLogger logger = Utils.getLogger( + KvmPhysicalServerResourceCascadeExtension.class); + private static final String NAME = "PhysicalServerComputeResourceAssignment"; + + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + + @Override + public void asyncCascade(CascadeAction action, Completion completion) { + if (physicalServerManager == null + || !HostVO.class.getSimpleName().equals(action.getParentIssuer()) + || !action.isActionCode( + CascadeConstant.DELETION_DELETE_CODE, + CascadeConstant.DELETION_FORCE_DELETE_CODE)) { + completion.success(); + return; + } + + List hosts = action.getParentIssuerContext(); + if (hosts == null || hosts.isEmpty()) { + completion.success(); + return; + } + boolean forceDelete = action.isActionCode(CascadeConstant.DELETION_FORCE_DELETE_CODE); + List forceDeleteFailures = Collections.synchronizedList(new ArrayList<>()); + + new While<>(hosts).each((host, each) -> { + if (host.getServerUuid() == null) { + each.done(); + return; + } + physicalServerManager.releaseResourceAssignment( + host.getServerUuid(), + KvmPhysicalServerAdapter.ROLE_TYPE, + host.getUuid(), + forceDelete, + new Completion(each) { + @Override + public void success() { + each.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + if (!forceDelete) { + each.addError(errorCode); + each.done(); + return; + } + forceDeleteFailures.add(String.format( + "host[uuid:%s]: %s", host.getUuid(), errorCode)); + logger.error(String.format( + "failed to release compute resource assignment before force deleting " + + "host[uuid:%s], continuing without recovery reconcile: %s", + host.getUuid(), errorCode)); + each.done(); + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!forceDelete && !errorCodeList.getCauses().isEmpty()) { + completion.fail(errorCodeList.getCauses().get(0)); + return; + } + if (!forceDeleteFailures.isEmpty()) { + logger.error(String.format( + "failed to release compute resource assignment for %s host(s) during " + + "force deletion; recovery reconcile was skipped: %s", + forceDeleteFailures.size(), String.join("; ", forceDeleteFailures))); + } + completion.success(); + } + }); + } + + @Override + public List getEdgeNames() { + return Collections.singletonList(HostVO.class.getSimpleName()); + } + + @Override + public String getCascadeResourceName() { + return NAME; + } + + @Override + public CascadeAction createActionForChildResource(CascadeAction action) { + return null; + } +} diff --git a/plugin/kvm/src/main/resources/physical-server-roles/compute.yaml b/plugin/kvm/src/main/resources/physical-server-roles/compute.yaml new file mode 100644 index 00000000000..3019a784339 --- /dev/null +++ b/plugin/kvm/src/main/resources/physical-server-roles/compute.yaml @@ -0,0 +1,60 @@ +roleType: COMPUTE +applicationMode: RESOURCE_HANDLES +sliceName: zstack-compute.slice +services: + - name: kvmagent + handleType: SYSTEMD_UNIT + value: zstack-kvmagent.service + required: true + restartable: false + - name: virtlogd + handleType: SYSTEMD_UNIT + value: virtlogd.service + required: false + restartable: false + - name: network-agent + handleType: SYSTEMD_UNIT + value: zstack-network-agent.service + required: false + restartable: false + - name: sharedblock-agent + handleType: SYSTEMD_UNIT + value: zstack-sharedblock-agent.service + required: false + restartable: false + - name: node-exporter + handleType: SYSTEMD_UNIT + value: node_exporter.service + required: false + restartable: true + auxiliary: true + - name: collectd + handleType: SYSTEMD_UNIT + value: collectd.service + required: false + restartable: true + auxiliary: true + - name: collectd-exporter + handleType: SYSTEMD_UNIT + value: collectd_exporter.service + required: false + restartable: true + auxiliary: true + - name: process-exporter + handleType: SYSTEMD_UNIT + value: process_exporter.service + required: false + restartable: true + auxiliary: true + - name: zstack-service-exporter + handleType: SYSTEMD_UNIT + value: zstack_service_exporter.service + required: false + restartable: true + auxiliary: true + - name: pushgateway + handleType: SYSTEMD_UNIT + value: pushgateway.service + required: false + restartable: true + auxiliary: true diff --git a/plugin/physicalServer/pom.xml b/plugin/physicalServer/pom.xml new file mode 100644 index 00000000000..3f7f96e718e --- /dev/null +++ b/plugin/physicalServer/pom.xml @@ -0,0 +1,75 @@ + + + 4.0.0 + + plugin + org.zstack + 5.5.0 + .. + + physicalServer + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${project.compiler.version} + + groovy-eclipse-compiler + ${project.java.version} + ${project.java.version} + lines,vars,source + true + + + + org.codehaus.groovy + groovy-eclipse-compiler + ${groovy.eclipse.compiler} + + + org.codehaus.groovy + groovy-eclipse-batch + ${groovy.eclipse.batch} + + + + + org.codehaus.mojo + aspectj-maven-plugin + ${aspectj.plugin.version} + + + + compile + test-compile + + + + + ${project.java.version} + ${project.java.version} + ${project.java.version} + true + + + org.springframework + spring-aspects + + + org.zstack + core + + + org.zstack + header + + + + + + + diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesMsg.java new file mode 100644 index 00000000000..530410c343e --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesMsg.java @@ -0,0 +1,35 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.message.APISyncCallMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; + +@Action(category = PhysicalServerConstant.ACTION_CATEGORY, names = {"read"}) +@RestRequest( + path = "/physical-servers/{serverUuid}/managed-services", + method = HttpMethod.GET, + responseClass = APIGetPhysicalServerManagedServicesReply.class +) +public class APIGetPhysicalServerManagedServicesMsg + extends APISyncCallMessage implements PhysicalServerMessage { + @APIParam(resourceType = PhysicalServerVO.class, operationTarget = true) + private String serverUuid; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public static APIGetPhysicalServerManagedServicesMsg __example__() { + APIGetPhysicalServerManagedServicesMsg msg = + new APIGetPhysicalServerManagedServicesMsg(); + msg.setServerUuid(uuid()); + return msg; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..d6cbdcc814c --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesMsgDoc_zh_cn.groovy @@ -0,0 +1,28 @@ +package org.zstack.physicalserver + +doc { + title "查看物理服务器管控服务资源使用(GetPhysicalServerManagedServices)" + category "物理服务器" + desc """返回物理服务器上各Role Manifest登记服务的运行事实。cpuSet为服务当前允许运行的CPU集合；cpuTime为cgroup累计CPU时间，单位纳秒；memory与memoryLimit单位均为字节，memoryLimit为包含父Role Slice约束后的有效上限，0表示不限。""" + + rest { + request { + url "GET /v1/physical-servers/{serverUuid}/managed-services" + header (Authorization: 'OAuth the-session-uuid') + clz APIGetPhysicalServerManagedServicesMsg.class + params { + column { + name "serverUuid" + desc "物理服务器UUID" + location "url" + type "String" + optional false + since "5.5.38" + } + } + } + response { + clz APIGetPhysicalServerManagedServicesReply.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesReply.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesReply.java new file mode 100644 index 00000000000..26d5fbc6d78 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIGetPhysicalServerManagedServicesReply.java @@ -0,0 +1,39 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.APIReply; +import org.zstack.header.rest.RestResponse; + +import java.util.ArrayList; +import java.util.List; + +@RestResponse(allTo = "services") +public class APIGetPhysicalServerManagedServicesReply extends APIReply { + private List services = + new ArrayList<>(); + + public List getServices() { + return services; + } + + public void setServices( + List services) { + this.services = services; + } + + public static APIGetPhysicalServerManagedServicesReply __example__() { + PhysicalServerManagedServiceInventory service = + new PhysicalServerManagedServiceInventory(); + service.setRoleType(PhysicalServerRoleType.COMPUTE); + service.setServiceName("node-exporter"); + service.setRestartable(true); + service.setState("RUNNING"); + service.setCpuSet("0-3"); + service.setCpuTime(12000000000L); + service.setMemory(96L * 1024 * 1024); + service.setMemoryLimit(4L * 1024 * 1024 * 1024); + APIGetPhysicalServerManagedServicesReply reply = + new APIGetPhysicalServerManagedServicesReply(); + reply.setServices(java.util.Collections.singletonList(service)); + return reply; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsg.java new file mode 100644 index 00000000000..a48d70aa082 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsg.java @@ -0,0 +1,25 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.query.APIQueryMessage; +import org.zstack.header.query.AutoQuery; +import org.zstack.header.rest.RestRequest; + +import java.util.List; + +import static java.util.Arrays.asList; + +@AutoQuery(replyClass = APIQueryPhysicalServerReply.class, inventoryClass = PhysicalServerInventory.class) +@Action(category = PhysicalServerConstant.ACTION_CATEGORY, names = {"read"}) +@RestRequest( + path = "/physical-servers", + optionalPaths = {"/physical-servers/{uuid}"}, + method = HttpMethod.GET, + responseClass = APIQueryPhysicalServerReply.class +) +public class APIQueryPhysicalServerMsg extends APIQueryMessage { + public static List __example__() { + return asList("uuid=" + uuid()); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..7547462f7a2 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsgDoc_zh_cn.groovy @@ -0,0 +1,31 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIQueryPhysicalServerReply +import org.zstack.header.query.APIQueryMessage + +doc { + title "查询物理服务器(QueryPhysicalServer)" + + category "物理服务器" + + desc """查询由计算节点或管理节点自动识别并关联的物理服务器。""" + + rest { + request { + url "GET /v1/physical-servers" + url "GET /v1/physical-servers/{uuid}" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIQueryPhysicalServerMsg.class + + desc """""" + + params APIQueryMessage.class + } + + response { + clz APIQueryPhysicalServerReply.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReply.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReply.java new file mode 100644 index 00000000000..a2e50d42f13 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReply.java @@ -0,0 +1,19 @@ +package org.zstack.physicalserver; + +import org.zstack.header.query.APIQueryReply; +import org.zstack.header.rest.RestResponse; + +import java.util.List; + +@RestResponse(allTo = "inventories") +public class APIQueryPhysicalServerReply extends APIQueryReply { + private List inventories; + + public List getInventories() { + return inventories; + } + + public void setInventories(List inventories) { + this.inventories = inventories; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReplyDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReplyDoc_zh_cn.groovy new file mode 100644 index 00000000000..bd38b3e1595 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReplyDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.PhysicalServerInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "查询物理服务器的返回" + + ref { + name "inventories" + path "org.zstack.physicalserver.APIQueryPhysicalServerReply.inventories" + desc "物理服务器清单" + type "List" + since "5.5.38" + clz PhysicalServerInventory.class + } + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIQueryPhysicalServerReply.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsg.java new file mode 100644 index 00000000000..201f92c8702 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsg.java @@ -0,0 +1,28 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.query.APIQueryMessage; +import org.zstack.header.query.AutoQuery; +import org.zstack.header.rest.RestRequest; + +import java.util.List; + +import static java.util.Arrays.asList; + +@AutoQuery( + replyClass = APIQueryPhysicalServerResourceAssignmentReply.class, + inventoryClass = PhysicalServerResourceAssignmentInventory.class +) +@Action(category = PhysicalServerConstant.ACTION_CATEGORY, names = {"read"}) +@RestRequest( + path = "/physical-servers/resource-assignments", + optionalPaths = {"/physical-servers/resource-assignments/{uuid}"}, + method = HttpMethod.GET, + responseClass = APIQueryPhysicalServerResourceAssignmentReply.class +) +public class APIQueryPhysicalServerResourceAssignmentMsg extends APIQueryMessage { + public static List __example__() { + return asList("serverUuid=" + uuid(), "roleType=ZBS"); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..11aa68df519 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy @@ -0,0 +1,31 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentReply +import org.zstack.header.query.APIQueryMessage + +doc { + title "查询物理服务器资源分配(QueryPhysicalServerResourceAssignment)" + + category "物理服务器" + + desc """查询物理服务器资源分配的期望、实际状态及展示态。""" + + rest { + request { + url "GET /v1/physical-servers/resource-assignments" + url "GET /v1/physical-servers/resource-assignments/{uuid}" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIQueryPhysicalServerResourceAssignmentMsg.class + + desc """""" + + params APIQueryMessage.class + } + + response { + clz APIQueryPhysicalServerResourceAssignmentReply.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReply.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReply.java new file mode 100644 index 00000000000..ac43e18a06c --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReply.java @@ -0,0 +1,19 @@ +package org.zstack.physicalserver; + +import org.zstack.header.query.APIQueryReply; +import org.zstack.header.rest.RestResponse; + +import java.util.List; + +@RestResponse(allTo = "inventories") +public class APIQueryPhysicalServerResourceAssignmentReply extends APIQueryReply { + private List inventories; + + public List getInventories() { + return inventories; + } + + public void setInventories(List inventories) { + this.inventories = inventories; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReplyDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReplyDoc_zh_cn.groovy new file mode 100644 index 00000000000..83502502dc3 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReplyDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "查询物理服务器资源分配的返回" + + ref { + name "inventories" + path "org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentReply.inventories" + desc "物理服务器资源分配清单" + type "List" + since "5.5.38" + clz PhysicalServerResourceAssignmentInventory.class + } + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentReply.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsEvent.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsEvent.java new file mode 100644 index 00000000000..5ba1eaa9fcb --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsEvent.java @@ -0,0 +1,14 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; + +@RestResponse +public class APIRefreshPhysicalServerResourceAssignmentsEvent extends APIEvent { + public APIRefreshPhysicalServerResourceAssignmentsEvent() { + } + + public APIRefreshPhysicalServerResourceAssignmentsEvent(String apiId) { + super(apiId); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsEventDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..ee97044cee0 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsEventDoc_zh_cn.groovy @@ -0,0 +1,23 @@ +package org.zstack.physicalserver + +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "刷新物理服务器资源分配的返回" + + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIRefreshPhysicalServerResourceAssignmentsEvent.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsMsg.java new file mode 100644 index 00000000000..b61d44c51d3 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsMsg.java @@ -0,0 +1,60 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; + +import java.util.ArrayList; +import java.util.List; + +@Action(category = PhysicalServerConstant.ACTION_CATEGORY) +@RestRequest( + path = "/physical-servers/{serverUuid}/resource-assignments/actions", + method = HttpMethod.PUT, + responseClass = APIRefreshPhysicalServerResourceAssignmentsEvent.class, + isAction = true +) +public class APIRefreshPhysicalServerResourceAssignmentsMsg extends APIMessage implements PhysicalServerMessage { + @APIParam(resourceType = PhysicalServerVO.class, operationTarget = true) + private String serverUuid; + + @APIParam(required = false, maxLength = 64) + private String roleType; + + @APIParam(required = false, maxLength = 64) + private List serviceNames = new ArrayList<>(); + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public List getServiceNames() { + return serviceNames; + } + + public void setServiceNames(List serviceNames) { + this.serviceNames = serviceNames; + } + + public static APIRefreshPhysicalServerResourceAssignmentsMsg __example__() { + APIRefreshPhysicalServerResourceAssignmentsMsg msg = + new APIRefreshPhysicalServerResourceAssignmentsMsg(); + msg.setServerUuid(uuid()); + return msg; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..f347346d6c1 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerResourceAssignmentsMsgDoc_zh_cn.groovy @@ -0,0 +1,81 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIRefreshPhysicalServerResourceAssignmentsEvent + +doc { + title "刷新物理服务器资源分配(RefreshPhysicalServerResourceAssignments)" + + category "物理服务器" + + desc """未指定serviceNames时，异步触发指定物理服务器全部Role的资源分配事实探测和收敛。指定roleType与serviceNames时，只重启Manifest中允许重启的目标服务；服务重启完成后触发资源分配重算。""" + + rest { + request { + url "PUT /v1/physical-servers/{serverUuid}/resource-assignments/actions" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIRefreshPhysicalServerResourceAssignmentsMsg.class + + desc """""" + + params { + + column { + name "serverUuid" + enclosedIn "refreshPhysicalServerResourceAssignments" + desc "物理服务器UUID" + location "url" + type "String" + optional false + since "5.5.38" + + } + column { + name "roleType" + enclosedIn "refreshPhysicalServerResourceAssignments" + desc "需要重启服务的Role；仅与serviceNames同时填写" + location "body" + type "String" + optional true + since "5.5.38" + + } + column { + name "serviceNames" + enclosedIn "refreshPhysicalServerResourceAssignments" + desc "需要重启的稳定服务名；必须来自管控服务查询结果且restartable为true" + location "body" + type "List" + optional true + since "5.5.38" + + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + } + } + + response { + clz APIRefreshPhysicalServerResourceAssignmentsEvent.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentEvent.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentEvent.java new file mode 100644 index 00000000000..0389f0d2514 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentEvent.java @@ -0,0 +1,24 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; + +@RestResponse(allTo = "inventory") +public class APIUpdatePhysicalServerResourceAssignmentEvent extends APIEvent { + private PhysicalServerResourceAssignmentInventory inventory; + + public APIUpdatePhysicalServerResourceAssignmentEvent() { + } + + public APIUpdatePhysicalServerResourceAssignmentEvent(String apiId) { + super(apiId); + } + + public PhysicalServerResourceAssignmentInventory getInventory() { + return inventory; + } + + public void setInventory(PhysicalServerResourceAssignmentInventory inventory) { + this.inventory = inventory; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentEventDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..ab4e9600f7d --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentEventDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "更新物理服务器资源分配的返回" + + ref { + name "inventory" + path "org.zstack.physicalserver.APIUpdatePhysicalServerResourceAssignmentEvent.inventory" + desc "已持久化的资源分配清单；实际状态可能仍在异步收敛" + type "PhysicalServerResourceAssignmentInventory" + since "5.5.38" + clz PhysicalServerResourceAssignmentInventory.class + } + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIUpdatePhysicalServerResourceAssignmentEvent.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentMsg.java new file mode 100644 index 00000000000..56b90d2e33e --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentMsg.java @@ -0,0 +1,71 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; + +@Action(category = PhysicalServerConstant.ACTION_CATEGORY) +@RestRequest( + path = "/physical-servers/{serverUuid}/resource-assignments/{roleType}/actions", + method = HttpMethod.PUT, + responseClass = APIUpdatePhysicalServerResourceAssignmentEvent.class, + isAction = true +) +public class APIUpdatePhysicalServerResourceAssignmentMsg extends APIMessage implements PhysicalServerMessage { + @APIParam(resourceType = PhysicalServerVO.class, operationTarget = true) + private String serverUuid; + + @APIParam(maxLength = 64) + private String roleType; + + @APIParam(required = false, maxLength = 4096) + private String cpuSet; + + @APIParam(required = false) + private Long memory; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public static APIUpdatePhysicalServerResourceAssignmentMsg __example__() { + APIUpdatePhysicalServerResourceAssignmentMsg msg = + new APIUpdatePhysicalServerResourceAssignmentMsg(); + msg.setServerUuid(uuid()); + msg.setRoleType(PhysicalServerRoleType.COMPUTE); + msg.setCpuSet("0-3"); + msg.setMemory(4L * 1024 * 1024 * 1024); + return msg; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..1eafa283bd6 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy @@ -0,0 +1,91 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIUpdatePhysicalServerResourceAssignmentEvent + +doc { + title "更新物理服务器资源分配(UpdatePhysicalServerResourceAssignment)" + + category "物理服务器" + + desc """增量更新指定物理服务器上一个Role的CPU和内存分配并异步触发同步。未填写的字段保持原值；同一物理服务器上的变更由服务端队列串行处理。ZBS Role会影响计算容量；COMPUTE和MANAGEMENT Role的软隔离不扣减计算容量。""" + + rest { + request { + url "PUT /v1/physical-servers/{serverUuid}/resource-assignments/{roleType}/actions" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIUpdatePhysicalServerResourceAssignmentMsg.class + + desc """""" + + params { + + column { + name "serverUuid" + enclosedIn "updatePhysicalServerResourceAssignment" + desc "物理服务器UUID" + location "url" + type "String" + optional false + since "5.5.38" + + } + column { + name "roleType" + enclosedIn "updatePhysicalServerResourceAssignment" + desc "节点Role：ZBS、MANAGEMENT或COMPUTE" + location "url" + type "String" + optional false + since "5.5.38" + + } + column { + name "cpuSet" + enclosedIn "updatePhysicalServerResourceAssignment" + desc "CPU集合，例如0-3,8-11；未填写表示保持原值" + location "body" + type "String" + optional true + since "5.5.38" + + } + column { + name "memory" + enclosedIn "updatePhysicalServerResourceAssignment" + desc "Role总体内存上限，单位字节且必须为1MiB的整数倍；0表示不限，未填写表示保持原值。更新先持久化Role Slice与服务归属配置，不主动重启服务；运行中服务可由刷新接口指定重启后进入该Slice" + location "body" + type "Long" + optional true + since "5.5.38" + + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + } + } + + response { + clz APIUpdatePhysicalServerResourceAssignmentEvent.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PackageInfo.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PackageInfo.java new file mode 100644 index 00000000000..79a1fe224d5 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PackageInfo.java @@ -0,0 +1,13 @@ +package org.zstack.physicalserver; + +import org.zstack.header.PackageAPIInfo; + +@PackageAPIInfo( + APICategoryName = "物理服务器", + permissions = { + PackageAPIInfo.PERMISSION_COMMUNITY_AVAILABLE, + PackageAPIInfo.PERMISSION_ZSV_BASIC_AVAILABLE + } +) +public class PackageInfo { +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerApiInterceptor.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerApiInterceptor.java new file mode 100644 index 00000000000..2472bb573d5 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerApiInterceptor.java @@ -0,0 +1,116 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.header.apimediator.ApiMessageInterceptionException; +import org.zstack.header.apimediator.ApiMessageInterceptor; +import org.zstack.header.message.APIMessage; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter; +import org.zstack.utils.data.SizeUnit; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.zstack.core.Platform.argerr; + +public class PhysicalServerApiInterceptor implements ApiMessageInterceptor { + @Autowired + private CloudBus bus; + @Autowired + private PluginRegistry pluginRgty; + + @Override + public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionException { + if (msg instanceof APIUpdatePhysicalServerResourceAssignmentMsg) { + validate((APIUpdatePhysicalServerResourceAssignmentMsg) msg); + } else if (msg instanceof APIRefreshPhysicalServerResourceAssignmentsMsg) { + validate((APIRefreshPhysicalServerResourceAssignmentsMsg) msg); + } + + if (msg instanceof PhysicalServerMessage) { + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + } + return msg; + } + + private void validate(APIRefreshPhysicalServerResourceAssignmentsMsg msg) { + List serviceNames = msg.getServiceNames(); + boolean restartServices = serviceNames != null && !serviceNames.isEmpty(); + if (!restartServices) { + if (msg.getRoleType() != null) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "SERVICE_NAMES_REQUIRED: roleType is only valid when serviceNames are specified")); + } + return; + } + if (msg.getRoleType() == null || adapters().get(msg.getRoleType()) == null) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "ROLE_TYPE_NOT_SUPPORTED: roleType[%s] is not registered", + msg.getRoleType())); + } + if (serviceNames.size() > 64) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "SERVICE_NAME_SET_INVALID: at most 64 services can be restarted")); + } + Set unique = new HashSet<>(); + for (String serviceName : serviceNames) { + if (serviceName == null + || !serviceName.matches("[A-Za-z0-9][A-Za-z0-9_.-]{0,63}") + || !unique.add(serviceName)) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "SERVICE_NAME_SET_INVALID: service names must be unique stable names")); + } + } + } + + private void validate(APIUpdatePhysicalServerResourceAssignmentMsg msg) { + PhysicalServerResourceControlAdapter adapter = + adapters().get(msg.getRoleType()); + if (adapter == null) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "ROLE_TYPE_NOT_SUPPORTED: roleType[%s] is not registered", msg.getRoleType())); + } + + if (msg.getCpuSet() == null && msg.getMemory() == null) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_UPDATE_EMPTY: cpuSet or memory must be specified")); + } + + if (msg.getMemory() != null) { + long mebibyte = SizeUnit.MEGABYTE.toByte(1); + if (msg.getMemory() < 0 || msg.getMemory() % mebibyte != 0) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "MEMORY_INVALID: memory[%s] must be 0 or a positive multiple of 1 MiB", + msg.getMemory())); + } + } + + if (msg.getCpuSet() == null) { + return; + } + try { + msg.setCpuSet(PhysicalServerCpuSet.normalize(msg.getCpuSet())); + } catch (IllegalArgumentException error) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "%s", error.getMessage())); + } + } + + private PhysicalServerResourceControlAdapterRegistry adapters() { + return PhysicalServerResourceControlAdapterRegistry.load(pluginRgty); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerAssignmentRepository.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerAssignmentRepository.java new file mode 100644 index 00000000000..537fe57ba21 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerAssignmentRepository.java @@ -0,0 +1,167 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQL; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.zstack.core.Platform.operr; + +public class PhysicalServerAssignmentRepository { + private static final CLogger logger = + Utils.getLogger(PhysicalServerAssignmentRepository.class); + + @Autowired + private DatabaseFacade dbf; + + @Transactional + public void ensureDefaults( + Collection serverUuids, String roleType) { + if (serverUuids == null || serverUuids.isEmpty()) { + return; + } + Query insert = dbf.getEntityManager().createNativeQuery( + "INSERT IGNORE INTO PhysicalServerResourceAssignmentVO " + + "(uuid, serverUuid, roleType, cpuSet, state, createDate, lastOpDate) " + + "SELECT REPLACE(UUID(), '-', ''), p.uuid, :roleType, '', :state, NOW(), NOW() " + + "FROM PhysicalServerVO p WHERE p.uuid IN (:serverUuids)"); + insert.setParameter("serverUuids", serverUuids); + insert.setParameter("roleType", roleType); + insert.setParameter("state", PhysicalServerResourceAssignmentState.Unsynced.name()); + insert.executeUpdate(); + } + + public PhysicalServerResourceAssignmentVO update( + APIUpdatePhysicalServerResourceAssignmentMsg msg) { + PhysicalServerResourceAssignmentVO current = requireAssignment( + msg.getServerUuid(), msg.getRoleType()); + String cpuSet = msg.getCpuSet() == null + ? current.getCpuSet() : msg.getCpuSet(); + Long memory = msg.getMemory() == null + ? current.getMemory() : msg.getMemory(); + int updated = SQL.New( + "update PhysicalServerResourceAssignmentVO a set " + + "a.cpuSet = :cpuSet, a.memory = :memory, a.state = :state " + + "where a.uuid = :uuid") + .param("cpuSet", cpuSet) + .param("memory", memory) + .param("state", PhysicalServerResourceAssignmentState.Unsynced) + .param("uuid", current.getUuid()) + .execute(); + if (updated == 0) { + throw assignmentMissing(msg.getServerUuid(), msg.getRoleType()); + } + PhysicalServerResourceAssignmentVO result = dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + logger.info(String.format( + "physical server resource assignment updated: serverUuid[%s], " + + "roleType[%s], accountUuid[%s], cpuSet[%s -> %s], memory[%s -> %s]", + current.getServerUuid(), current.getRoleType(), + msg.getSession() == null ? null : msg.getSession().getAccountUuid(), + current.getCpuSet(), result.getCpuSet(), + current.getMemory(), result.getMemory())); + return result; + } + + public PhysicalServerResourceAssignmentVO initializeCpuSet( + PhysicalServerResourceAssignmentVO current, String cpuSet) { + if (current.getCpuSet() != null && !current.getCpuSet().isEmpty()) { + return current; + } + SQL.New( + "update PhysicalServerResourceAssignmentVO a set " + + "a.cpuSet = :cpuSet, a.state = :state " + + "where a.uuid = :uuid and a.cpuSet = ''") + .param("cpuSet", cpuSet) + .param("state", PhysicalServerResourceAssignmentState.Unsynced) + .param("uuid", current.getUuid()) + .execute(); + return dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + } + + public boolean markSynced(PhysicalServerResourceAssignmentVO applied) { + String memoryPredicate = applied.getMemory() == null + ? "a.memory is null" : "a.memory = :memory"; + SQL sql = SQL.New( + "update PhysicalServerResourceAssignmentVO a set a.state = :state " + + "where a.uuid = :uuid and a.cpuSet = :cpuSet and " + memoryPredicate) + .param("state", PhysicalServerResourceAssignmentState.Synced) + .param("uuid", applied.getUuid()) + .param("cpuSet", applied.getCpuSet()); + if (applied.getMemory() != null) { + sql.param("memory", applied.getMemory()); + } + return sql.execute() == 1; + } + + public void markUnsynced(String uuid) { + SQL.New("update PhysicalServerResourceAssignmentVO a " + + "set a.state = :state where a.uuid = :uuid") + .param("state", PhysicalServerResourceAssignmentState.Unsynced) + .param("uuid", uuid) + .execute(); + } + + public boolean delete(String uuid) { + boolean deleted = SQL.New( + "delete from PhysicalServerResourceAssignmentVO a where a.uuid = :uuid") + .param("uuid", uuid) + .execute() == 1; + if (deleted) { + logger.info(String.format( + "physical server resource assignment deleted after release: assignmentUuid[%s]", + uuid)); + } + return deleted; + } + + public PhysicalServerResourceAssignmentVO find( + String serverUuid, String roleType) { + return Q.New(PhysicalServerResourceAssignmentVO.class) + .eq(PhysicalServerResourceAssignmentVO_.serverUuid, serverUuid) + .eq(PhysicalServerResourceAssignmentVO_.roleType, roleType) + .find(); + } + + public List listAssignments() { + return Q.New(PhysicalServerResourceAssignmentVO.class).list(); + } + + public List listAssignments( + Collection serverUuids) { + if (serverUuids == null || serverUuids.isEmpty()) { + return Collections.emptyList(); + } + return Q.New(PhysicalServerResourceAssignmentVO.class) + .in(PhysicalServerResourceAssignmentVO_.serverUuid, serverUuids) + .list(); + } + + private PhysicalServerResourceAssignmentVO requireAssignment( + String serverUuid, String roleType) { + PhysicalServerResourceAssignmentVO current = find(serverUuid, roleType); + if (current == null) { + throw assignmentMissing(serverUuid, roleType); + } + return current; + } + + private OperationFailureException assignmentMissing( + String serverUuid, String roleType) { + return new OperationFailureException(operr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_NOT_FOUND: resource assignment for role[%s] " + + "does not exist on physical server[uuid:%s]", + roleType, serverUuid)); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerConstant.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerConstant.java new file mode 100644 index 00000000000..a2501ad7b2d --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerConstant.java @@ -0,0 +1,9 @@ +package org.zstack.physicalserver; + +public interface PhysicalServerConstant { + String SERVICE_ID = "physicalServer"; + String ACTION_CATEGORY = "physicalServer"; + String CONTROL_OWNER_KEY = "physical-server-control-owner-v1"; + String ERROR_CODE = "ORG_ZSTACK_PHYSICALSERVER_10000"; + long RECONCILE_INTERVAL_MILLIS = 100_000L; +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlanner.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlanner.java new file mode 100644 index 00000000000..2579943bc3a --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlanner.java @@ -0,0 +1,55 @@ +package org.zstack.physicalserver; + +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.SortedSet; + +public class PhysicalServerCpuPlanner { + public String validateAndNormalize( + PhysicalServerResourceIsolationMode isolationMode, + String cpuSet, + PhysicalServerCpuTopology topology, + Collection allocatedExclusiveCpus) { + SortedSet desired = PhysicalServerCpuSet.parse( + cpuSet, topology.getOnlineCpus()); + if (!disjoint(desired, allocatedExclusiveCpus)) { + throw new IllegalArgumentException(String.format( + "CPU_SET_CONFLICT: %s CPU set overlaps an exclusive role", + isolationMode == PhysicalServerResourceIsolationMode.EXCLUSIVE + ? "exclusive" : "shared")); + } + if (isolationMode == PhysicalServerResourceIsolationMode.EXCLUSIVE) { + validateWholeCores(desired, topology); + } + return PhysicalServerCpuSet.format(desired); + } + + private void validateWholeCores( + Set desired, + PhysicalServerCpuTopology topology) { + for (PhysicalServerCpuTopology.CoreGroup group : topology.getCoreGroups()) { + boolean intersects = !disjoint(desired, group.getCpus()); + if (intersects && !desired.containsAll(group.getCpus())) { + throw new IllegalArgumentException(String.format( + "CPU_SIBLING_SPLIT: CPU_SET splits core group[%s]", + PhysicalServerCpuSet.format(group.getCpus()))); + } + } + if (!disjoint(desired, topology.getCpuZeroGroup().getCpus())) { + throw new IllegalArgumentException( + "CPU_ZERO_RESERVED: CPU0 core group must remain shared"); + } + } + + private boolean disjoint( + Collection left, Collection right) { + Set copy = new HashSet<>(left); + copy.retainAll(right); + return copy.isEmpty(); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerIdentityService.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerIdentityService.java new file mode 100644 index 00000000000..1dd601b3448 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerIdentityService.java @@ -0,0 +1,205 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.core.Platform; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import javax.persistence.Tuple; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PhysicalServerIdentityService { + private static final CLogger logger = Utils.getLogger(PhysicalServerIdentityService.class); + + @Autowired + private DatabaseFacade dbf; + + @Transactional + public Map resolve(Collection identities) { + Map requestedZones = normalize(identities); + if (requestedZones.isEmpty()) { + return Collections.emptyMap(); + } + + Map servers = serversBySerial(requestedZones.keySet()); + Set missing = new LinkedHashSet<>(requestedZones.keySet()); + missing.removeAll(servers.keySet()); + insertServers(missing); + servers = serversBySerial(requestedZones.keySet()); + assignMissingZones(servers, requestedZones); + servers = serversBySerial(requestedZones.keySet()); + + Map result = new LinkedHashMap<>(); + for (Map.Entry requested : requestedZones.entrySet()) { + IdentityRow server = servers.get(requested.getKey()); + if (server == null || (requested.getValue() != null + && !requested.getValue().equals(server.zoneUuid))) { + logger.warn(String.format( + "cannot resolve physical server identity for serialNumber[%s], " + + "reasonCode[IDENTITY_ZONE_CONFLICT]: serverZoneUuid[%s], requestedZoneUuid[%s]", + requested.getKey(), server == null ? null : server.zoneUuid, + requested.getValue())); + continue; + } + result.put(requested.getKey(), server.uuid); + } + return result; + } + + public Map findSerialNumbersByServerUuids( + Collection serverUuids) { + if (serverUuids == null || serverUuids.isEmpty()) { + return Collections.emptyMap(); + } + Map result = new LinkedHashMap<>(); + for (Tuple server : Q.New(PhysicalServerVO.class) + .select(PhysicalServerVO_.uuid, PhysicalServerVO_.serialNumber) + .in(PhysicalServerVO_.uuid, serverUuids) + .listTuple()) { + result.put(server.get(0, String.class), server.get(1, String.class)); + } + return result; + } + + private Map normalize(Collection identities) { + if (identities == null || identities.isEmpty()) { + return Collections.emptyMap(); + } + Map requestedZones = new LinkedHashMap<>(); + Set conflicts = new LinkedHashSet<>(); + for (PhysicalServerIdentitySpec identity : identities) { + if (identity == null) { + continue; + } + String serialNumber = Platform.normalizeMachineSerialNumber(identity.getSerialNumber()); + if (serialNumber == null) { + continue; + } + String previousZone = requestedZones.get(serialNumber); + String requestedZone = identity.getZoneUuid(); + if (previousZone != null && requestedZone != null && !previousZone.equals(requestedZone)) { + conflicts.add(serialNumber); + continue; + } + if (!requestedZones.containsKey(serialNumber) || previousZone == null) { + requestedZones.put(serialNumber, requestedZone); + } + } + for (String serialNumber : conflicts) { + requestedZones.remove(serialNumber); + logger.warn(String.format( + "cannot resolve physical server identity for serialNumber[%s], " + + "reasonCode[IDENTITY_AMBIGUOUS]: multiple zones reported", + serialNumber)); + } + return requestedZones; + } + + private Map serversBySerial(Collection serialNumbers) { + if (serialNumbers.isEmpty()) { + return Collections.emptyMap(); + } + Map result = new HashMap<>(); + List servers = Q.New(PhysicalServerVO.class) + .select(PhysicalServerVO_.uuid, PhysicalServerVO_.serialNumber, + PhysicalServerVO_.zoneUuid) + .in(PhysicalServerVO_.serialNumber, serialNumbers) + .listTuple(); + for (Tuple server : servers) { + IdentityRow row = new IdentityRow( + server.get(0, String.class), server.get(2, String.class)); + result.put(server.get(1, String.class), row); + } + return result; + } + + private void insertServers(Collection serialNumbers) { + if (serialNumbers.isEmpty()) { + return; + } + StringBuilder sql = new StringBuilder( + "INSERT IGNORE INTO PhysicalServerVO " + + "(uuid, serialNumber, createDate, lastOpDate) VALUES "); + int index = 0; + for (String ignored : serialNumbers) { + if (index > 0) { + sql.append(','); + } + sql.append("(:uuid").append(index).append(", :serial").append(index) + .append(", NOW(), NOW())"); + index++; + } + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + index = 0; + for (String serialNumber : serialNumbers) { + query.setParameter("uuid" + index, Platform.getUuid()); + query.setParameter("serial" + index, serialNumber); + index++; + } + query.executeUpdate(); + } + + private void assignMissingZones( + Map servers, + Map requestedZones) { + Map zonesByServer = new LinkedHashMap<>(); + for (Map.Entry requested : requestedZones.entrySet()) { + IdentityRow server = servers.get(requested.getKey()); + if (server != null && server.zoneUuid == null + && requested.getValue() != null) { + zonesByServer.put(server.uuid, requested.getValue()); + } + } + if (zonesByServer.isEmpty()) { + return; + } + + StringBuilder sql = new StringBuilder( + "UPDATE PhysicalServerVO SET zoneUuid = CASE uuid "); + int index = 0; + for (String ignored : zonesByServer.keySet()) { + sql.append("WHEN :uuid").append(index) + .append(" THEN :zone").append(index).append(' '); + index++; + } + sql.append("END WHERE zoneUuid IS NULL AND uuid IN ("); + for (index = 0; index < zonesByServer.size(); index++) { + if (index > 0) { + sql.append(','); + } + sql.append(":uuid").append(index); + } + sql.append(')'); + + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + index = 0; + for (Map.Entry entry : zonesByServer.entrySet()) { + query.setParameter("uuid" + index, entry.getKey()); + query.setParameter("zone" + index, entry.getValue()); + index++; + } + query.executeUpdate(); + } + + private static class IdentityRow { + private final String uuid; + private final String zoneUuid; + + private IdentityRow(String uuid, String zoneUuid) { + this.uuid = uuid; + this.zoneUuid = zoneUuid; + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventory.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventory.java new file mode 100644 index 00000000000..e12b6f60e68 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventory.java @@ -0,0 +1,78 @@ +package org.zstack.physicalserver; + +import org.zstack.header.configuration.PythonClassInventory; +import org.zstack.header.search.Inventory; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Inventory(mappingVOClass = PhysicalServerVO.class) +@PythonClassInventory +public class PhysicalServerInventory implements Serializable { + private String uuid; + private String zoneUuid; + private String serialNumber; + private Timestamp createDate; + private Timestamp lastOpDate; + + public static PhysicalServerInventory valueOf(PhysicalServerVO vo) { + PhysicalServerInventory inventory = new PhysicalServerInventory(); + inventory.setUuid(vo.getUuid()); + inventory.setZoneUuid(vo.getZoneUuid()); + inventory.setSerialNumber(vo.getSerialNumber()); + inventory.setCreateDate(vo.getCreateDate()); + inventory.setLastOpDate(vo.getLastOpDate()); + return inventory; + } + + public static List valueOf(Collection vos) { + List inventories = new ArrayList<>(vos.size()); + for (PhysicalServerVO vo : vos) { + inventories.add(valueOf(vo)); + } + return inventories; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getZoneUuid() { + return zoneUuid; + } + + public void setZoneUuid(String zoneUuid) { + this.zoneUuid = zoneUuid; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventoryDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventoryDoc_zh_cn.groovy new file mode 100644 index 00000000000..c87fa226ced --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventoryDoc_zh_cn.groovy @@ -0,0 +1,39 @@ +package org.zstack.physicalserver + +import java.sql.Timestamp + +doc { + + title "物理服务器清单" + + field { + name "uuid" + desc "资源的UUID，唯一标示该资源" + type "String" + since "5.5.38" + } + field { + name "zoneUuid" + desc "区域UUID" + type "String" + since "5.5.38" + } + field { + name "serialNumber" + desc "经标准化的服务器序列号，在云平台范围内唯一" + type "String" + since "5.5.38" + } + field { + name "createDate" + desc "创建时间" + type "Timestamp" + since "5.5.38" + } + field { + name "lastOpDate" + desc "最后一次修改时间" + type "Timestamp" + since "5.5.38" + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagedServiceInventory.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagedServiceInventory.java new file mode 100644 index 00000000000..7559824e0a5 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagedServiceInventory.java @@ -0,0 +1,108 @@ +package org.zstack.physicalserver; + +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class PhysicalServerManagedServiceInventory implements Serializable { + private String roleType; + private String serviceName; + private boolean restartable; + private String state; + private String cpuSet; + private Long cpuTime; + private Long memory; + private Long memoryLimit; + + public static PhysicalServerManagedServiceInventory valueOf( + ManagedServiceResourceUsage usage) { + PhysicalServerManagedServiceInventory inventory = + new PhysicalServerManagedServiceInventory(); + inventory.setRoleType(usage.getRoleType()); + inventory.setServiceName(usage.getServiceName()); + inventory.setRestartable(usage.isRestartable()); + inventory.setState(usage.getState()); + inventory.setCpuSet(usage.getCpuSet()); + inventory.setCpuTime(usage.getCpuTime()); + inventory.setMemory(usage.getMemory()); + inventory.setMemoryLimit(usage.getMemoryLimit()); + return inventory; + } + + public static List valueOf( + Collection usages) { + List result = + new ArrayList<>(); + for (ManagedServiceResourceUsage usage : usages) { + result.add(valueOf(usage)); + } + return result; + } + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public boolean isRestartable() { + return restartable; + } + + public void setRestartable(boolean restartable) { + this.restartable = restartable; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getCpuTime() { + return cpuTime; + } + + public void setCpuTime(Long cpuTime) { + this.cpuTime = cpuTime; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public Long getMemoryLimit() { + return memoryLimit; + } + + public void setMemoryLimit(Long memoryLimit) { + this.memoryLimit = memoryLimit; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagerImpl.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagerImpl.java new file mode 100644 index 00000000000..c2d3a4869da --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagerImpl.java @@ -0,0 +1,286 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.MessageSafe; +import org.zstack.core.cloudbus.ResourceDestinationMaker; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.thread.PeriodicTask; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.header.AbstractService; +import org.zstack.header.Component; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +public class PhysicalServerManagerImpl extends AbstractService implements + PhysicalServerManager, + Component, + ManagementNodeReadyExtensionPoint { + @Autowired + private CloudBus bus; + @Autowired + private PhysicalServerIdentityService identity; + @Autowired + private PhysicalServerResourceAssignmentReconciler reconciler; + @Autowired + private ThreadFacade thdf; + @Autowired + private ResourceDestinationMaker destinationMaker; + @Autowired + private PluginRegistry pluginRgty; + private Future reconcileTask; + + @Override + @MessageSafe + public void handleMessage(Message msg) { + if (msg instanceof APIMessage) { + handleApiMessage((APIMessage) msg); + } else if (msg instanceof ReconcilePhysicalServerMsg) { + ReconcilePhysicalServerMsg rmsg = (ReconcilePhysicalServerMsg) msg; + reconciler.enqueue(rmsg.getServerUuid(), rmsg.isRefreshFacts()); + } else if (msg instanceof ReconcileAllPhysicalServersMsg) { + reconciler.enqueueAll(); + } else if (msg instanceof ReleasePhysicalServerResourceAssignmentMsg) { + handle((ReleasePhysicalServerResourceAssignmentMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handle(ReleasePhysicalServerResourceAssignmentMsg msg) { + MessageReply reply = new MessageReply(); + reconciler.releaseAssignment( + msg.getServerUuid(), + msg.getRoleType(), + msg.getConsumerUuid(), + msg.isForce(), + new Completion(msg) { + @Override + public void success() { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handleApiMessage(APIMessage msg) { + if (msg instanceof APIUpdatePhysicalServerResourceAssignmentMsg) { + handle((APIUpdatePhysicalServerResourceAssignmentMsg) msg); + } else if (msg instanceof APIRefreshPhysicalServerResourceAssignmentsMsg) { + handle((APIRefreshPhysicalServerResourceAssignmentsMsg) msg); + } else if (msg instanceof APIGetPhysicalServerManagedServicesMsg) { + handle((APIGetPhysicalServerManagedServicesMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handle( + APIGetPhysicalServerManagedServicesMsg msg) { + APIGetPhysicalServerManagedServicesReply reply = + new APIGetPhysicalServerManagedServicesReply(); + reconciler.collectManagedServiceUsage( + msg.getServerUuid(), + new ReturnValueCompletion>(msg) { + @Override + public void success( + List usages) { + reply.setServices( + PhysicalServerManagedServiceInventory.valueOf(usages)); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(APIUpdatePhysicalServerResourceAssignmentMsg msg) { + APIUpdatePhysicalServerResourceAssignmentEvent event = + new APIUpdatePhysicalServerResourceAssignmentEvent(msg.getId()); + reconciler.updateAssignment( + msg, + new ReturnValueCompletion(msg) { + @Override + public void success(PhysicalServerResourceAssignmentInventory inventory) { + event.setInventory(inventory); + reconcile(msg.getServerUuid(), true); + bus.publish(event); + } + + @Override + public void fail(ErrorCode errorCode) { + event.setError(errorCode); + bus.publish(event); + } + }); + } + + private void handle(APIRefreshPhysicalServerResourceAssignmentsMsg msg) { + APIRefreshPhysicalServerResourceAssignmentsEvent event = + new APIRefreshPhysicalServerResourceAssignmentsEvent(msg.getId()); + if (msg.getServiceNames() != null && !msg.getServiceNames().isEmpty()) { + reconciler.restartManagedServices(msg, new Completion(msg) { + @Override + public void success() { + bus.publish(event); + } + + @Override + public void fail(ErrorCode errorCode) { + event.setError(errorCode); + bus.publish(event); + } + }); + return; + } + reconcile(msg.getServerUuid(), true); + bus.publish(event); + } + + @Override + public Map resolveIdentities( + Collection identities) { + return identity.resolve(identities); + } + + @Override + public Map findSerialNumbersByServerUuids( + Collection serverUuids) { + return identity.findSerialNumbersByServerUuids(serverUuids); + } + + @Override + public void ensureResourceAssignments( + Collection serverUuids, String roleType) { + reconciler.ensureResourceAssignments(serverUuids, roleType); + } + + @Override + public void reconcile(String serverUuid, boolean refreshFacts) { + if (serverUuid == null) { + return; + } + ReconcilePhysicalServerMsg msg = new ReconcilePhysicalServerMsg(); + msg.setServerUuid(serverUuid); + msg.setRefreshFacts(refreshFacts); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg); + } + + @Override + public void reconcileAll() { + ReconcileAllPhysicalServersMsg msg = new ReconcileAllPhysicalServersMsg(); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg); + } + + @Override + public void releaseResourceAssignment( + String serverUuid, + String roleType, + String consumerUuid, + boolean force, + Completion completion) { + ReleasePhysicalServerResourceAssignmentMsg msg = + new ReleasePhysicalServerResourceAssignmentMsg(); + msg.setServerUuid(serverUuid); + msg.setRoleType(roleType); + msg.setConsumerUuid(consumerUuid); + msg.setForce(force); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + completion.success(); + } + }); + } + + @Override + public boolean start() { + return true; + } + + @Override + public void managementNodeReady() { + reconcileAllIfOwned(); + reconcileTask = thdf.submitPeriodicTask(new PeriodicTask() { + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.MILLISECONDS; + } + + @Override + public long getInterval() { + return PhysicalServerConstant.RECONCILE_INTERVAL_MILLIS; + } + + @Override + public String getName() { + return "physical-server-resource-assignment-reconcile"; + } + + @Override + public void run() { + reconcileAllIfOwned(); + } + }); + } + + private void reconcileAllIfOwned() { + if (destinationMaker.isManagedByUs(PhysicalServerConstant.CONTROL_OWNER_KEY)) { + reconcileAll(); + } + } + + @Override + public boolean stop() { + if (reconcileTask != null) { + reconcileTask.cancel(true); + } + return true; + } + + @Override + public String getId() { + return bus.makeLocalServiceId(PhysicalServerConstant.SERVICE_ID); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerMessage.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerMessage.java new file mode 100644 index 00000000000..b0162c74c43 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerMessage.java @@ -0,0 +1,5 @@ +package org.zstack.physicalserver; + +public interface PhysicalServerMessage { + String getServerUuid(); +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerReasonCodes.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerReasonCodes.java new file mode 100644 index 00000000000..ad1ec23e84a --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerReasonCodes.java @@ -0,0 +1,31 @@ +package org.zstack.physicalserver; + +import org.zstack.header.errorcode.ErrorCode; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class PhysicalServerReasonCodes { + private static final Pattern REASON_CODE = Pattern.compile( + "(?:^|[^A-Z0-9_])([A-Z][A-Z0-9_]+):"); + + private PhysicalServerReasonCodes() { + } + + public static String from(ErrorCode errorCode) { + String details = errorCode == null ? null : errorCode.getDetails(); + return from(details); + } + + public static String from(String details) { + if (details == null) { + return "RESOURCE_ASSIGNMENT_INVALID"; + } + Matcher matcher = REASON_CODE.matcher(details); + String reasonCode = null; + while (matcher.find()) { + reasonCode = matcher.group(1); + } + return reasonCode == null ? "RESOURCE_ASSIGNMENT_INVALID" : reasonCode; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventory.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventory.java new file mode 100644 index 00000000000..6b3cc836830 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventory.java @@ -0,0 +1,112 @@ +package org.zstack.physicalserver; + +import org.zstack.header.configuration.PythonClassInventory; +import org.zstack.header.search.Inventory; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Inventory(mappingVOClass = PhysicalServerResourceAssignmentVO.class) +@PythonClassInventory +public class PhysicalServerResourceAssignmentInventory implements Serializable { + private String uuid; + private String serverUuid; + private String roleType; + private String cpuSet; + private Long memory; + private String state; + private Timestamp createDate; + private Timestamp lastOpDate; + + public static PhysicalServerResourceAssignmentInventory valueOf( + PhysicalServerResourceAssignmentVO vo) { + PhysicalServerResourceAssignmentInventory inventory = + new PhysicalServerResourceAssignmentInventory(); + inventory.setUuid(vo.getUuid()); + inventory.setServerUuid(vo.getServerUuid()); + inventory.setRoleType(vo.getRoleType()); + inventory.setCpuSet(vo.getCpuSet()); + inventory.setMemory(vo.getMemory()); + inventory.setState(vo.getState().name()); + inventory.setCreateDate(vo.getCreateDate()); + inventory.setLastOpDate(vo.getLastOpDate()); + return inventory; + } + + public static List valueOf( + Collection vos) { + List inventories = + new ArrayList<>(vos.size()); + for (PhysicalServerResourceAssignmentVO vo : vos) { + inventories.add(valueOf(vo)); + } + return inventories; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventoryDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventoryDoc_zh_cn.groovy new file mode 100644 index 00000000000..9a0da6fa34c --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventoryDoc_zh_cn.groovy @@ -0,0 +1,54 @@ +package org.zstack.physicalserver + +doc { + title "物理服务器资源分配清单" + + field { + name "uuid" + desc "资源分配UUID" + type "String" + since "5.5.38" + } + field { + name "serverUuid" + desc "物理服务器UUID" + type "String" + since "5.5.38" + } + field { + name "roleType" + desc "资源使用角色，例如ZBS、MANAGEMENT或COMPUTE" + type "String" + since "5.5.38" + } + field { + name "cpuSet" + desc "分配给该Role的CPU集合，例如0-3,8-11" + type "String" + since "5.5.38" + } + field { + name "memory" + desc "应用到该Role每个服务Handle的内存上限，单位字节；0表示不限，空表示不控制内存" + type "Long" + since "5.5.38" + } + field { + name "state" + desc "资源分配状态：Unsynced表示尚未完成同步，Synced表示全部服务Handle已应用并校验成功" + type "String" + since "5.5.38" + } + field { + name "createDate" + desc "创建时间" + type "Timestamp" + since "5.5.38" + } + field { + name "lastOpDate" + desc "最后一次修改时间" + type "Timestamp" + since "5.5.38" + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentReconciler.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentReconciler.java new file mode 100644 index 00000000000..4a02108a683 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentReconciler.java @@ -0,0 +1,739 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.thread.ChainTask; +import org.zstack.core.thread.SingleFlightTask; +import org.zstack.core.thread.SyncTaskChain; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerResourceConsumerState; +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter; +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode; +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static org.zstack.core.Platform.operr; + +public class PhysicalServerResourceAssignmentReconciler { + private static final long RECONCILE_REQUEUE_DELAY_MILLIS = 100; + private static final long MAX_REFRESH_RETRY_DELAY_MILLIS = + TimeUnit.SECONDS.toMillis(30); + private static final CLogger logger = Utils.getLogger( + PhysicalServerResourceAssignmentReconciler.class); + + private final ConcurrentMap reconcileQueues = + new ConcurrentHashMap<>(); + private final ReconcileQueueState reconcileAllQueue = new ReconcileQueueState(); + private final ConcurrentMap pendingRefreshServers = + new ConcurrentHashMap<>(); + private final AtomicBoolean refreshBatchScheduled = new AtomicBoolean(); + private final AtomicInteger refreshBatchFailures = new AtomicInteger(); + private final AtomicLong refreshSequence = new AtomicLong(); + + @Autowired + private ThreadFacade thdf; + @Autowired + private PluginRegistry pluginRgty; + @Autowired + private PhysicalServerAssignmentRepository assignments; + @Autowired + private PhysicalServerCpuPlanner planner; + @Autowired + private PhysicalServerResourceControlReconciler resourceControls; + + public void ensureResourceAssignments( + Collection serverUuids, + String roleType) { + PhysicalServerResourceControlAdapter adapter = adapters().get(roleType); + if (adapter == null) { + throw new IllegalArgumentException(String.format( + "ROLE_TYPE_NOT_SUPPORTED: roleType[%s]", roleType)); + } + assignments.ensureDefaults( + serverUuids, + roleType); + } + + public void enqueueAll() { + reconcileAllQueue.dirty.set(true); + scheduleQueuedReconcileAll(); + } + + private void scheduleQueuedReconcileAll() { + if (!reconcileAllQueue.scheduled.compareAndSet(false, true)) { + return; + } + reconcileAllQueue.dirty.set(false); + thdf.singleFlightSubmit(new SingleFlightTask(null) + .setSyncSignature( + "reconcile-all-physical-server-resource-assignment") + .run(completion -> { + refreshAndEnqueueAll(); + completion.success(null); + }) + .done(result -> { + if (!result.isSuccess()) { + logger.warn(String.format( + "failed to refresh all physical server resource assignments: %s", + result.getErrorCode())); + } + reconcileAllQueue.scheduled.set(false); + if (reconcileAllQueue.dirty.get()) { + thdf.submitTimeoutTask( + this::scheduleQueuedReconcileAll, + TimeUnit.MILLISECONDS, + RECONCILE_REQUEUE_DELAY_MILLIS); + } + })); + } + + private void refreshAndEnqueueAll() { + PhysicalServerResourceControlAdapterRegistry adapters = adapters(); + Set serverUuids = new HashSet<>(); + for (PhysicalServerResourceControlAdapter adapter : + adapters.orderedAdapters()) { + try { + adapter.refreshAssociations(Collections.emptySet()); + Set associated = adapter.getAssociatedServerUuids(); + serverUuids.addAll(associated); + assignments.ensureDefaults( + adapter.getEligibleDefaultServerUuids(), + adapter.getRoleType()); + } catch (RuntimeException error) { + logger.warn(String.format( + "failed to refresh resource assignment adapter: " + + "roleType[%s], error[%s]", + adapter.getRoleType(), error.getMessage())); + } + } + for (PhysicalServerResourceAssignmentVO assignment : + assignments.listAssignments()) { + serverUuids.add(assignment.getServerUuid()); + } + for (String serverUuid : serverUuids) { + enqueue(serverUuid, false); + } + } + + public void enqueue(String serverUuid, boolean refreshFacts) { + if (refreshFacts) { + pendingRefreshServers.put( + serverUuid, refreshSequence.incrementAndGet()); + scheduleRefreshBatch(RECONCILE_REQUEUE_DELAY_MILLIS); + return; + } + ReconcileQueueState queue = reconcileQueues.computeIfAbsent( + serverUuid, key -> new ReconcileQueueState()); + queue.dirty.set(true); + scheduleQueuedReconcile(serverUuid, queue); + } + + private void scheduleRefreshBatch(long delayMillis) { + if (!refreshBatchScheduled.compareAndSet(false, true)) { + return; + } + thdf.submitTimeoutTask( + this::submitRefreshBatch, + TimeUnit.MILLISECONDS, + delayMillis); + } + + private void submitRefreshBatch() { + thdf.singleFlightSubmit(new SingleFlightTask(null) + .setSyncSignature( + "refresh-physical-server-resource-assignment-facts") + .run(completion -> { + if (refreshAndEnqueuePendingServers()) { + completion.success(null); + } else { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_FACT_REFRESH_FAILED")); + } + }) + .done(result -> { + long delay = RECONCILE_REQUEUE_DELAY_MILLIS; + if (result.isSuccess()) { + refreshBatchFailures.set(0); + } else { + delay = nextRefreshRetryDelay(); + logger.warn(String.format( + "failed to refresh physical server resource assignment facts: %s", + result.getErrorCode())); + } + refreshBatchScheduled.set(false); + if (!pendingRefreshServers.isEmpty()) { + scheduleRefreshBatch(delay); + } + })); + } + + private long nextRefreshRetryDelay() { + int failures = Math.min(refreshBatchFailures.incrementAndGet(), 8); + long base = Math.min( + MAX_REFRESH_RETRY_DELAY_MILLIS, + RECONCILE_REQUEUE_DELAY_MILLIS << (failures - 1)); + long upper = Math.min(MAX_REFRESH_RETRY_DELAY_MILLIS, base * 2); + return upper == base + ? base : ThreadLocalRandom.current().nextLong(base, upper + 1); + } + + private boolean refreshAndEnqueuePendingServers() { + Map pending = new HashMap<>(pendingRefreshServers); + if (pending.isEmpty()) { + return true; + } + Set serverUuids = pending.keySet(); + boolean refreshed = true; + for (PhysicalServerResourceControlAdapter adapter : + adapters().orderedAdapters()) { + try { + adapter.refreshAssociations(serverUuids); + Map states = + adapter.getStates(serverUuids); + Set eligible = new HashSet<>( + adapter.getEligibleDefaultServerUuids()); + eligible.retainAll(serverUuids); + for (Map.Entry state : + states.entrySet()) { + if (state.getValue() + == PhysicalServerResourceConsumerState.MISSING) { + eligible.remove(state.getKey()); + } + } + assignments.ensureDefaults( + eligible, + adapter.getRoleType()); + } catch (RuntimeException error) { + refreshed = false; + logger.warn(String.format( + "failed to refresh resource assignment adapter: " + + "roleType[%s], error[%s]", + adapter.getRoleType(), error.getMessage())); + } + } + + if (refreshed) { + for (Map.Entry entry : pending.entrySet()) { + pendingRefreshServers.remove(entry.getKey(), entry.getValue()); + } + } + for (String serverUuid : serverUuids) { + enqueue(serverUuid, false); + } + return refreshed; + } + + public void updateAssignment( + APIUpdatePhysicalServerResourceAssignmentMsg msg, + ReturnValueCompletion completion) { + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + return serverOperationSignature(msg.getServerUuid()); + } + + @Override + public void run(SyncTaskChain chain) { + validateUpdate(msg, new Completion(completion) { + @Override + public void success() { + try { + PhysicalServerResourceAssignmentVO updated = + assignments.update(msg); + completion.success( + PhysicalServerResourceAssignmentInventory.valueOf(updated)); + } catch (OperationFailureException error) { + completion.fail(error.getErrorCode()); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "%s", error.getMessage())); + } finally { + chain.next(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + try { + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }); + } + + @Override + public String getName() { + return String.format( + "update-physical-server-resource-assignment-%s-%s", + msg.getServerUuid(), msg.getRoleType()); + } + }); + } + + public void collectManagedServiceUsage( + String serverUuid, + ReturnValueCompletion> completion) { + List current = + assignments.listAssignments(Collections.singleton(serverUuid)); + Set roleTypes = current.stream() + .map(PhysicalServerResourceAssignmentVO::getRoleType) + .collect(java.util.stream.Collectors.toSet()); + List ordered = new ArrayList<>(roleTypes); + ordered.sort(String::compareTo); + collectManagedServiceUsage( + serverUuid, ordered, roleTypes, 0, + new ArrayList<>(), completion); + } + + private void collectManagedServiceUsage( + String serverUuid, + List roleTypes, + Set assignedRoleTypes, + int index, + List result, + ReturnValueCompletion> completion) { + if (index == roleTypes.size()) { + completion.success(result); + return; + } + String roleType = roleTypes.get(index); + PhysicalServerResourceControlAdapter adapter = adapters().get(roleType); + if (adapter == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_ADAPTER_MISSING: roleType[%s]", + roleType)); + return; + } + boolean includeAuxiliaryServices = + PhysicalServerRoleType.MANAGEMENT.equals(roleType) + || !assignedRoleTypes.contains( + PhysicalServerRoleType.MANAGEMENT); + adapter.collectManagedServiceUsage( + serverUuid, + includeAuxiliaryServices, + new ReturnValueCompletion>(completion) { + @Override + public void success(List services) { + if (services != null) { + for (ManagedServiceResourceUsage service : services) { + service.setRoleType(roleType); + result.add(service); + } + } + collectManagedServiceUsage( + serverUuid, roleTypes, assignedRoleTypes, + index + 1, result, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + public void restartManagedServices( + APIRefreshPhysicalServerResourceAssignmentsMsg msg, + Completion completion) { + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + return serverOperationSignature(msg.getServerUuid()); + } + + @Override + public void run(SyncTaskChain chain) { + Map snapshot = + assignmentsByRole(assignments.listAssignments( + Collections.singleton(msg.getServerUuid()))) + .getOrDefault( + msg.getServerUuid(), + Collections.emptyMap()); + PhysicalServerResourceAssignmentVO assignment = + snapshot.get(msg.getRoleType()); + PhysicalServerResourceControlAdapter adapter = + adapters().get(msg.getRoleType()); + if (assignment == null || adapter == null) { + try { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_NOT_FOUND: resource assignment for role[%s] does not exist on physical server[uuid:%s]", + msg.getRoleType(), msg.getServerUuid())); + } finally { + chain.next(); + } + return; + } + boolean hasManagement = snapshot.containsKey( + PhysicalServerRoleType.MANAGEMENT); + boolean includeAuxiliaryServices = + PhysicalServerRoleType.MANAGEMENT.equals(msg.getRoleType()) + || !hasManagement; + resourceControls.stageForServiceRestart( + assignment, + snapshot, + adapter, + new Completion(completion) { + @Override + public void success() { + restartManagedServicesAfterStage( + msg, + adapter, + includeAuxiliaryServices, + chain, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + try { + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }); + } + + @Override + public String getName() { + return String.format( + "restart-physical-server-managed-services-%s-%s", + msg.getServerUuid(), msg.getRoleType()); + } + }); + } + + private void restartManagedServicesAfterStage( + APIRefreshPhysicalServerResourceAssignmentsMsg msg, + PhysicalServerResourceControlAdapter adapter, + boolean includeAuxiliaryServices, + SyncTaskChain chain, + Completion completion) { + adapter.restartManagedServices( + msg.getServerUuid(), + includeAuxiliaryServices, + msg.getServiceNames(), + new Completion(completion) { + @Override + public void success() { + try { + enqueue(msg.getServerUuid(), true); + completion.success(); + } finally { + chain.next(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + try { + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }); + } + + private void validateUpdate( + APIUpdatePhysicalServerResourceAssignmentMsg msg, + Completion completion) { + PhysicalServerResourceControlAdapterRegistry adapters = adapters(); + PhysicalServerResourceControlAdapter adapter = + adapters.get(msg.getRoleType()); + if (adapter == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "ROLE_TYPE_NOT_SUPPORTED: roleType[%s]", msg.getRoleType())); + return; + } + PhysicalServerResourceAssignmentVO existing = assignments.find( + msg.getServerUuid(), msg.getRoleType()); + if (existing == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_NOT_FOUND: resource assignment for role[%s] " + + "does not exist on physical server[uuid:%s]", + msg.getRoleType(), msg.getServerUuid())); + return; + } + if (msg.getCpuSet() == null) { + completion.success(); + return; + } + PhysicalServerResourceControlAdapter topologyAdapter = + adapters.get(adapter.getTopologyRoleType()); + if (topologyAdapter == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "CPU_TOPOLOGY_SOURCE_MISSING: roleType[%s] requires topologyRoleType[%s]", + msg.getRoleType(), adapter.getTopologyRoleType())); + return; + } + Map snapshot = + assignmentsByRole(assignments.listAssignments( + Collections.singleton(msg.getServerUuid()))) + .getOrDefault(msg.getServerUuid(), Collections.emptyMap()); + try { + topologyAdapter.collectTopology( + msg.getServerUuid(), + new ReturnValueCompletion(completion) { + @Override + public void success(PhysicalServerCpuTopology topology) { + try { + String normalized = planner.validateAndNormalize( + adapter.getIsolationMode(), + msg.getCpuSet(), + topology, + reservedExclusiveCpus( + existing, + snapshot.values(), + adapters, + topology)); + msg.setCpuSet(normalized); + completion.success(); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "%s", error.getMessage())); + } + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "CPU_TOPOLOGY_EXECUTOR_UNREACHABLE: %s", + error.getMessage())); + } + } + + public void releaseAssignment( + String serverUuid, + String roleType, + String consumerUuid, + boolean force, + Completion completion) { + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + return serverOperationSignature(serverUuid); + } + + @Override + public void run(SyncTaskChain chain) { + resourceControls.release( + serverUuid, + roleType, + consumerUuid, + new Completion(completion) { + @Override + public void success() { + try { + completion.success(); + } finally { + chain.next(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + try { + if (force) { + resourceControls.forget( + serverUuid, roleType); + } else { + enqueue(serverUuid, false); + } + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }); + } + + @Override + public String getName() { + return String.format( + "release-physical-server-resource-assignment-%s-%s", + serverUuid, roleType); + } + }); + } + + private void scheduleQueuedReconcile( + String serverUuid, + ReconcileQueueState queue) { + if (!queue.scheduled.compareAndSet(false, true)) { + return; + } + queue.dirty.set(false); + submitReconcile(serverUuid, new Completion(null) { + @Override + public void success() { + completeQueuedReconcile(serverUuid, queue); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to reconcile resource assignment for " + + "physical server[uuid:%s]: %s", + serverUuid, errorCode)); + completeQueuedReconcile(serverUuid, queue); + } + }); + } + + private void completeQueuedReconcile( + String serverUuid, + ReconcileQueueState queue) { + queue.scheduled.set(false); + if (queue.dirty.get()) { + thdf.submitTimeoutTask( + () -> scheduleQueuedReconcile(serverUuid, queue), + TimeUnit.MILLISECONDS, + RECONCILE_REQUEUE_DELAY_MILLIS); + } + } + + private void submitReconcile( + String serverUuid, + Completion completion) { + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + return serverOperationSignature(serverUuid); + } + + @Override + public void run(SyncTaskChain chain) { + Map current = + assignmentsByRole(assignments.listAssignments( + Collections.singleton(serverUuid))) + .getOrDefault(serverUuid, Collections.emptyMap()); + resourceControls.reconcile( + serverUuid, + current, + new Completion(completion) { + @Override + public void success() { + try { + completion.success(); + } finally { + chain.next(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + try { + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }); + } + + @Override + public String getName() { + return String.format( + "reconcile-physical-server-resource-assignment-%s", + serverUuid); + } + }); + } + + private Set reservedExclusiveCpus( + PhysicalServerResourceAssignmentVO current, + Collection assignmentValues, + PhysicalServerResourceControlAdapterRegistry adapters, + PhysicalServerCpuTopology topology) { + Set result = new HashSet<>(); + for (PhysicalServerResourceAssignmentVO assignment : assignmentValues) { + if (current != null && current.getUuid().equals(assignment.getUuid())) { + continue; + } + PhysicalServerResourceControlAdapter adapter = + adapters.get(assignment.getRoleType()); + if (adapter == null || adapter.getIsolationMode() + != PhysicalServerResourceIsolationMode.EXCLUSIVE) { + continue; + } + String reserved = assignment.getCpuSet(); + if (reserved != null && !reserved.isEmpty()) { + result.addAll(PhysicalServerCpuSet.parse( + reserved, topology.getOnlineCpus())); + } + } + return result; + } + + private String serverOperationSignature(String serverUuid) { + return String.format( + "physical-server-resource-assignment-operation-%s", + serverUuid); + } + + private PhysicalServerResourceControlAdapterRegistry adapters() { + return PhysicalServerResourceControlAdapterRegistry.load(pluginRgty); + } + + private Map> assignmentsByServer( + Collection rows) { + Map> result = + new LinkedHashMap<>(); + for (PhysicalServerResourceAssignmentVO row : rows) { + result.computeIfAbsent( + row.getServerUuid(), ignored -> new LinkedHashMap<>()) + .put(row.getRoleType(), row); + } + return result; + } + + private Map> assignmentsByRole( + Collection rows) { + return assignmentsByServer(rows); + } + + private static class ReconcileQueueState { + private final AtomicBoolean scheduled = new AtomicBoolean(); + private final AtomicBoolean dirty = new AtomicBoolean(); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentState.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentState.java new file mode 100644 index 00000000000..0f3152c7b0b --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentState.java @@ -0,0 +1,6 @@ +package org.zstack.physicalserver; + +public enum PhysicalServerResourceAssignmentState { + Unsynced, + Synced +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO.java new file mode 100644 index 00000000000..7ed7c933c46 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO.java @@ -0,0 +1,119 @@ +package org.zstack.physicalserver; + +import org.zstack.header.vo.ForeignKey; +import org.zstack.header.vo.Index; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.PreUpdate; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import java.sql.Timestamp; + +@Entity +@Table(uniqueConstraints = @UniqueConstraint( + name = "ukPhysicalServerResourceAssignment", + columnNames = {"serverUuid", "roleType"} +)) +public class PhysicalServerResourceAssignmentVO { + @Id + @Column + @Index + private String uuid; + + @Column + @Index + @ForeignKey(parentEntityClass = PhysicalServerVO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String serverUuid; + + @Column + private String roleType; + + @Column(nullable = false, length = 4096) + private String cpuSet; + + @Column + private Long memory; + + @Column(nullable = false) + @Enumerated(EnumType.STRING) + private PhysicalServerResourceAssignmentState state; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + @PreUpdate + private void preUpdate() { + lastOpDate = null; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public PhysicalServerResourceAssignmentState getState() { + return state; + } + + public void setState(PhysicalServerResourceAssignmentState state) { + this.state = state; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO_.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO_.java new file mode 100644 index 00000000000..1e9efd86386 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO_.java @@ -0,0 +1,17 @@ +package org.zstack.physicalserver; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(PhysicalServerResourceAssignmentVO.class) +public class PhysicalServerResourceAssignmentVO_ { + public static volatile SingularAttribute uuid; + public static volatile SingularAttribute serverUuid; + public static volatile SingularAttribute roleType; + public static volatile SingularAttribute cpuSet; + public static volatile SingularAttribute memory; + public static volatile SingularAttribute state; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceControlAdapterRegistry.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceControlAdapterRegistry.java new file mode 100644 index 00000000000..df5f6a6d16a --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceControlAdapterRegistry.java @@ -0,0 +1,159 @@ +package org.zstack.physicalserver; + +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class PhysicalServerResourceControlAdapterRegistry { + private static final CLogger logger = Utils.getLogger( + PhysicalServerResourceControlAdapterRegistry.class); + + private final Map adapters; + private final Map errors; + + private PhysicalServerResourceControlAdapterRegistry( + Map adapters, + Map errors) { + this.adapters = adapters; + this.errors = errors; + } + + public static PhysicalServerResourceControlAdapterRegistry load( + PluginRegistry pluginRegistry) { + Map> candidates = + new LinkedHashMap<>(); + for (PhysicalServerResourceControlAdapter adapter : + pluginRegistry.getExtensionList( + PhysicalServerResourceControlAdapter.class)) { + String roleType = roleType(adapter); + if (roleType != null) { + candidates.computeIfAbsent( + roleType, ignored -> new ArrayList<>()).add(adapter); + } + } + + Map adapters = + new LinkedHashMap<>(); + Map errors = new LinkedHashMap<>(); + for (Map.Entry> entry : + candidates.entrySet()) { + String error; + try { + error = validate(entry.getKey(), entry.getValue()); + } catch (RuntimeException failure) { + error = String.format( + "RESOURCE_ASSIGNMENT_ADAPTER_INVALID: roleType[%s], error[%s]", + entry.getKey(), failure.getMessage()); + } + if (error == null) { + adapters.put(entry.getKey(), entry.getValue().get(0)); + } else { + errors.put(entry.getKey(), error); + logger.error(error); + } + } + boolean removed; + do { + removed = false; + for (Map.Entry entry : + new ArrayList<>(adapters.entrySet())) { + String topologyRoleType = entry.getValue().getTopologyRoleType(); + if (!adapters.containsKey(topologyRoleType)) { + String error = String.format( + "RESOURCE_ASSIGNMENT_TOPOLOGY_ROLE_UNAVAILABLE: roleType[%s], topologyRoleType[%s]", + entry.getKey(), topologyRoleType); + adapters.remove(entry.getKey()); + errors.put(entry.getKey(), error); + logger.error(error); + removed = true; + } + } + } while (removed); + return new PhysicalServerResourceControlAdapterRegistry(adapters, errors); + } + + private static String roleType(PhysicalServerResourceControlAdapter adapter) { + String value; + try { + value = adapter.getRoleType(); + } catch (RuntimeException error) { + logger.error(String.format( + "failed to read role type from resource assignment adapter[%s]: %s", + adapter.getClass().getName(), error.getMessage())); + return null; + } + if (value == null || !value.trim().matches("[A-Z][A-Z0-9_]{0,63}")) { + logger.error(String.format( + "resource assignment adapter[%s] returned invalid roleType[%s]", + adapter.getClass().getName(), value)); + return null; + } + return value.trim(); + } + + private static String validate( + String roleType, + List candidates) { + if (candidates.size() > 1) { + return String.format( + "RESOURCE_ASSIGNMENT_ADAPTER_AMBIGUOUS: %s adapters are registered for roleType[%s]", + candidates.size(), roleType); + } + PhysicalServerResourceControlAdapter adapter = candidates.get(0); + if (adapter.getIsolationMode() == null) { + return String.format( + "RESOURCE_ASSIGNMENT_ISOLATION_MODE_MISSING: roleType[%s]", + roleType); + } + if (adapter.getApplicationMode() == null) { + return String.format( + "RESOURCE_ASSIGNMENT_APPLICATION_MODE_MISSING: roleType[%s]", + roleType); + } + if (adapter.getTopologyRoleType() == null + || adapter.getTopologyRoleType().trim().isEmpty()) { + return String.format( + "RESOURCE_ASSIGNMENT_TOPOLOGY_ROLE_MISSING: roleType[%s]", + roleType); + } + return null; + } + + public PhysicalServerResourceControlAdapter get(String roleType) { + return adapters.get(roleType); + } + + public String getError(String roleType) { + return errors.get(roleType); + } + + public boolean contains(String roleType) { + return adapters.containsKey(roleType); + } + + public List orderedRoleTypes() { + List result = new ArrayList<>(adapters.keySet()); + result.sort(String::compareTo); + return result; + } + + public List orderedAdapters() { + List result = + new ArrayList<>(adapters.values()); + result.sort(Comparator.comparing( + PhysicalServerResourceControlAdapter::getRoleType)); + return result; + } + + public Collection errors() { + return errors.values(); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceControlReconciler.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceControlReconciler.java new file mode 100644 index 00000000000..8c4b1cdce15 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceControlReconciler.java @@ -0,0 +1,651 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerResourceApplicationMode; +import org.zstack.header.physicalserver.PhysicalServerResourceConsumerState; +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter; +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode; +import org.zstack.header.physicalserver.ResourceControlCommand; +import org.zstack.header.physicalserver.ResourceControlResponse; +import org.zstack.header.physicalserver.ResourceControlResult; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.zstack.core.Platform.operr; + +public class PhysicalServerResourceControlReconciler { + private static final CLogger logger = Utils.getLogger( + PhysicalServerResourceControlReconciler.class); + + @Autowired + private PluginRegistry pluginRgty; + @Autowired + private PhysicalServerAssignmentRepository assignments; + @Autowired + private PhysicalServerCpuPlanner planner; + + public void reconcile( + String serverUuid, + Map assignmentSnapshot, + Completion completion) { + PhysicalServerResourceControlAdapterRegistry adapters = adapters(); + Map before = capacityFootprint( + assignmentSnapshot.values(), adapters); + List roleTypes = new ArrayList<>(assignmentSnapshot.keySet()); + roleTypes.sort(Comparator + .comparingInt((String roleType) -> isolationOrder(adapters.get(roleType))) + .thenComparing(String::compareTo)); + reconcileRoles( + serverUuid, assignmentSnapshot, adapters, roleTypes, + 0, null, before, completion); + } + + private void reconcileRoles( + String serverUuid, + Map assignmentSnapshot, + PhysicalServerResourceControlAdapterRegistry adapters, + List roleTypes, + int index, + ErrorCode firstError, + Map before, + Completion completion) { + if (index == roleTypes.size()) { + if (!before.equals(capacityFootprint( + assignmentSnapshot.values(), adapters))) { + refreshCapacities(serverUuid); + } + if (firstError == null) { + completion.success(); + } else { + completion.fail(firstError); + } + return; + } + + String roleType = roleTypes.get(index); + reconcileRole( + assignmentSnapshot.get(roleType), assignmentSnapshot, adapters, + new Completion(completion) { + @Override + public void success() { + reconcileRoles( + serverUuid, assignmentSnapshot, adapters, + roleTypes, index + 1, firstError, before, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to reconcile physical server resource assignment: " + + "serverUuid[%s], roleType[%s], error[%s]", + serverUuid, roleType, errorCode)); + reconcileRoles( + serverUuid, assignmentSnapshot, adapters, + roleTypes, index + 1, + firstError == null ? errorCode : firstError, + before, completion); + } + }); + } + + private void reconcileRole( + PhysicalServerResourceAssignmentVO assignment, + Map assignmentSnapshot, + PhysicalServerResourceControlAdapterRegistry adapters, + Completion completion) { + PhysicalServerResourceControlAdapter adapter = + adapters.get(assignment.getRoleType()); + if (adapter == null) { + failUnsynced( + assignment, + adapterUnavailableReason(assignment.getRoleType(), adapters), + completion); + return; + } + + PhysicalServerResourceConsumerState state; + try { + state = adapter.getState(assignment.getServerUuid()); + } catch (RuntimeException error) { + failUnsynced(assignment, "RESOURCE_CONSUMER_UNAVAILABLE", completion); + return; + } + if (state != PhysicalServerResourceConsumerState.AVAILABLE) { + failUnsynced( + assignment, + state == PhysicalServerResourceConsumerState.MISSING + ? "RESOURCE_CONSUMER_MISSING" + : unavailableReason(adapter, assignment.getServerUuid()), + completion); + return; + } + + collectTopology( + assignment, adapter, adapters, + new ReturnValueCompletion(completion) { + @Override + public void success(PhysicalServerCpuTopology topology) { + validateAndApply( + assignment, assignmentSnapshot, adapter, + topology, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + failUnsynced( + assignment, + PhysicalServerReasonCodes.from(errorCode), + completion); + } + }); + } + + private void collectTopology( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerResourceControlAdapter adapter, + PhysicalServerResourceControlAdapterRegistry adapters, + ReturnValueCompletion completion) { + PhysicalServerResourceControlAdapter topologyAdapter = + adapters.get(adapter.getTopologyRoleType()); + if (topologyAdapter == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "CPU_TOPOLOGY_SOURCE_MISSING: roleType[%s] requires topologyRoleType[%s]", + assignment.getRoleType(), adapter.getTopologyRoleType())); + return; + } + PhysicalServerResourceConsumerState state; + try { + state = topologyAdapter.getState(assignment.getServerUuid()); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "CPU_TOPOLOGY_SOURCE_FAILED: %s", error.getMessage())); + return; + } + if (state != PhysicalServerResourceConsumerState.AVAILABLE) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + state == PhysicalServerResourceConsumerState.MISSING + ? "CPU_TOPOLOGY_SOURCE_MISSING" + : "CPU_TOPOLOGY_EXECUTOR_UNREACHABLE")); + return; + } + try { + topologyAdapter.collectTopology(assignment.getServerUuid(), completion); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "CPU_TOPOLOGY_EXECUTOR_UNREACHABLE: %s", error.getMessage())); + } + } + + private void validateAndApply( + PhysicalServerResourceAssignmentVO assignment, + Map assignmentSnapshot, + PhysicalServerResourceControlAdapter adapter, + PhysicalServerCpuTopology topology, + Completion completion) { + Set allocatedExclusiveCpus = allocatedExclusiveCpus( + assignment, assignmentSnapshot.values(), adapters(), topology); + PhysicalServerResourceAssignmentVO current = assignment; + try { + String cpuSet = current.getCpuSet(); + if (cpuSet == null || cpuSet.isEmpty()) { + cpuSet = adapter.getDefaultCpuSet(topology, allocatedExclusiveCpus); + } + cpuSet = planner.validateAndNormalize( + adapter.getIsolationMode(), cpuSet, + topology, allocatedExclusiveCpus); + if (!cpuSet.equals(current.getCpuSet())) { + current = assignments.initializeCpuSet(current, cpuSet); + if (current == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_NOT_FOUND: assignmentUuid[%s]", + assignment.getUuid())); + return; + } + assignmentSnapshot.put(current.getRoleType(), current); + } + } catch (RuntimeException error) { + failUnsynced( + assignment, + PhysicalServerReasonCodes.from(error.getMessage()), + completion); + return; + } + apply(current, assignmentSnapshot, adapter, topology, "APPLY", null, completion); + } + + public void release( + String serverUuid, + String roleType, + String consumerUuid, + Completion completion) { + PhysicalServerResourceAssignmentVO assignment = + assignments.find(serverUuid, roleType); + if (assignment == null) { + completion.success(); + return; + } + PhysicalServerResourceControlAdapter adapter = adapters().get(roleType); + if (adapter == null) { + failUnsynced(assignment, "RESOURCE_ASSIGNMENT_ADAPTER_MISSING", completion); + return; + } + apply( + assignment, assignmentsByRole(serverUuid), adapter, + null, "RELEASE", consumerUuid, completion); + } + + public void forget(String serverUuid, String roleType) { + PhysicalServerResourceAssignmentVO assignment = + assignments.find(serverUuid, roleType); + if (assignment != null && assignments.delete(assignment.getUuid())) { + refreshCapacities(serverUuid); + } + } + + public void stageForServiceRestart( + PhysicalServerResourceAssignmentVO assignment, + Map assignmentSnapshot, + PhysicalServerResourceControlAdapter adapter, + Completion completion) { + if (adapter.getApplicationMode() + != PhysicalServerResourceApplicationMode.RESOURCE_HANDLES) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "SERVICE_RESTART_NOT_SUPPORTED: roleType[%s] is provider-managed", + assignment.getRoleType())); + return; + } + ResourceControlCommand command = command( + assignment, + assignmentSnapshot.get(PhysicalServerRoleType.MANAGEMENT), + adapter, + "APPLY"); + try { + adapter.apply( + assignment.getServerUuid(), null, command, + new ReturnValueCompletion(completion) { + @Override + public void success(ResourceControlResponse response) { + if (stageMatches(command, response)) { + completion.success(); + return; + } + failUnsynced( + assignment, + "RESOURCE_CONTROL_STAGE_MISMATCH", + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + failUnsynced( + assignment, + PhysicalServerReasonCodes.from(errorCode), + completion); + } + }); + } catch (RuntimeException error) { + failUnsynced( + assignment, + PhysicalServerReasonCodes.from(error.getMessage()), + completion); + } + } + + private boolean stageMatches( + ResourceControlCommand command, + ResourceControlResponse response) { + if (response == null) { + return false; + } + try { + if (!command.getCpuSet().equals( + normalizeCpuSet(response.getCpuSet(), null))) { + return false; + } + } catch (RuntimeException error) { + return false; + } + return memoryMatches(response, command.getMemory()); + } + + private void apply( + PhysicalServerResourceAssignmentVO assignment, + Map assignmentSnapshot, + PhysicalServerResourceControlAdapter adapter, + PhysicalServerCpuTopology topology, + String operation, + String consumerUuid, + Completion completion) { + ResourceControlCommand command = command( + assignment, + assignmentSnapshot.get(PhysicalServerRoleType.MANAGEMENT), + adapter, + operation); + try { + adapter.apply( + assignment.getServerUuid(), consumerUuid, command, + new ReturnValueCompletion(completion) { + @Override + public void success(ResourceControlResponse response) { + if ("RELEASE".equals(operation)) { + completeRelease( + assignment, adapter, command, + response, completion); + } else { + completeApply( + assignment, assignmentSnapshot, + adapter, topology, command, + response, completion); + } + } + + @Override + public void fail(ErrorCode errorCode) { + failUnsynced( + assignment, + PhysicalServerReasonCodes.from(errorCode), + completion); + } + }); + } catch (RuntimeException error) { + failUnsynced( + assignment, + PhysicalServerReasonCodes.from(error.getMessage()), + completion); + } + } + + private void completeApply( + PhysicalServerResourceAssignmentVO assignment, + Map assignmentSnapshot, + PhysicalServerResourceControlAdapter adapter, + PhysicalServerCpuTopology topology, + ResourceControlCommand command, + ResourceControlResponse response, + Completion completion) { + if (!matches(adapter, command, response, topology, false)) { + failUnsynced(assignment, "RESOURCE_CONTROL_ACTUAL_MISMATCH", completion); + return; + } + if (!assignments.markSynced(assignment)) { + completion.success(); + return; + } + assignment.setState(PhysicalServerResourceAssignmentState.Synced); + assignmentSnapshot.put(assignment.getRoleType(), assignment); + completion.success(); + } + + private void completeRelease( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerResourceControlAdapter adapter, + ResourceControlCommand command, + ResourceControlResponse response, + Completion completion) { + if (!matches(adapter, command, response, null, true)) { + failUnsynced(assignment, "RESOURCE_CONTROL_RELEASE_FAILED", completion); + return; + } + if (!assignments.delete(assignment.getUuid())) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_ASSIGNMENT_NOT_FOUND: assignmentUuid[%s]", + assignment.getUuid())); + return; + } + refreshCapacities(assignment.getServerUuid()); + completion.success(); + } + + private boolean matches( + PhysicalServerResourceControlAdapter adapter, + ResourceControlCommand command, + ResourceControlResponse response, + PhysicalServerCpuTopology topology, + boolean release) { + if (response == null) { + return false; + } + String desiredCpuSet = release ? "" : command.getCpuSet(); + String actualCpuSet; + try { + actualCpuSet = normalizeCpuSet(response.getCpuSet(), topology); + } catch (RuntimeException error) { + return false; + } + if (!desiredCpuSet.equals(actualCpuSet)) { + return false; + } + Long desiredMemory = command.getMemory() == null + ? null : release ? 0L : command.getMemory(); + if (!memoryMatches(response, desiredMemory)) { + return false; + } + if (adapter.getApplicationMode() + == PhysicalServerResourceApplicationMode.PROVIDER_MANAGED) { + return release + ? "DISABLED".equals(response.getState()) + : "READY".equals(response.getState()); + } + return coverageMatches(response, release) + && handleResultsMatch(response, desiredCpuSet, desiredMemory, release); + } + + private boolean coverageMatches( + ResourceControlResponse response, boolean release) { + if (response.getExpectedServiceCount() == null + || response.getCoveredServiceCount() == null) { + return false; + } + return response.getExpectedServiceCount().equals( + response.getCoveredServiceCount()) + && (release || response.getExpectedServiceCount() > 0); + } + + private boolean memoryMatches( + ResourceControlResponse response, Long desiredMemory) { + return desiredMemory == null + || desiredMemory.equals( + response.getMemory()) + || desiredMemory == 0L + && response.getMemory() == null; + } + + private boolean handleResultsMatch( + ResourceControlResponse response, + String desiredCpuSet, + Long desiredMemory, + boolean release) { + if (response.getResults() == null + || response.getResults().isEmpty()) { + return release && Integer.valueOf(0).equals( + response.getExpectedServiceCount()); + } + int observed = 0; + for (ResourceControlResult result : response.getResults()) { + if ("SKIPPED".equals(result.getState())) { + continue; + } + observed++; + String requiredState = release ? "DISABLED" : "READY"; + if (!requiredState.equals(result.getState()) + || !desiredCpuSet.equals(normalizeEmpty(result.getCpuSet())) + || desiredMemory != null + && !desiredMemory.equals(result.getMemory()) + && !(desiredMemory == 0L + && result.getMemory() == null)) { + return false; + } + } + return observed == response.getExpectedServiceCount(); + } + + private ResourceControlCommand command( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerResourceAssignmentVO managementAssignment, + PhysicalServerResourceControlAdapter adapter, + String operation) { + ResourceControlCommand command = new ResourceControlCommand(); + command.setRoleType(assignment.getRoleType()); + command.setIsolationMode(adapter.getIsolationMode().name()); + command.setOperation(operation); + command.setCpuSet("RELEASE".equals(operation) ? "" : assignment.getCpuSet()); + command.setMemory(assignment.getMemory()); + command.setIncludeAuxiliaryServices( + PhysicalServerRoleType.MANAGEMENT.equals(assignment.getRoleType()) + || managementAssignment == null); + return command; + } + + private Set allocatedExclusiveCpus( + PhysicalServerResourceAssignmentVO current, + Collection assignmentValues, + PhysicalServerResourceControlAdapterRegistry adapters, + PhysicalServerCpuTopology topology) { + Set result = new HashSet<>(); + for (PhysicalServerResourceAssignmentVO assignment : assignmentValues) { + if (assignment.getUuid().equals(current.getUuid())) { + continue; + } + PhysicalServerResourceControlAdapter adapter = + adapters.get(assignment.getRoleType()); + if (adapter == null || adapter.getIsolationMode() + != PhysicalServerResourceIsolationMode.EXCLUSIVE + || assignment.getCpuSet() == null + || assignment.getCpuSet().isEmpty()) { + continue; + } + result.addAll(PhysicalServerCpuSet.parse( + assignment.getCpuSet(), topology.getOnlineCpus())); + } + return result; + } + + private String normalizeCpuSet( + String cpuSet, PhysicalServerCpuTopology topology) { + if (normalizeEmpty(cpuSet).isEmpty()) { + return ""; + } + if (topology == null) { + return PhysicalServerCpuSet.normalize(cpuSet); + } + return PhysicalServerCpuSet.format(PhysicalServerCpuSet.parse( + cpuSet, topology.getOnlineCpus())); + } + + private String normalizeEmpty(String value) { + return value == null ? "" : value.trim(); + } + + private void failUnsynced( + PhysicalServerResourceAssignmentVO assignment, + String reason, + Completion completion) { + assignments.markUnsynced(assignment.getUuid()); + assignment.setState(PhysicalServerResourceAssignmentState.Unsynced); + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "%s: physical server resource assignment[uuid:%s, roleType:%s] is unsynced", + reason, assignment.getUuid(), assignment.getRoleType())); + } + + private void refreshCapacities(String serverUuid) { + for (PhysicalServerResourceControlAdapter adapter : + adapters().orderedAdapters()) { + try { + adapter.refreshCapacity(serverUuid); + } catch (RuntimeException error) { + logger.warn(String.format( + "failed to refresh capacity after resource assignment changed: " + + "serverUuid[%s], roleType[%s], error[%s]", + serverUuid, adapter.getRoleType(), error.getMessage())); + } + } + } + + private Map assignmentsByRole( + String serverUuid) { + Map result = + new LinkedHashMap<>(); + for (PhysicalServerResourceAssignmentVO assignment : + assignments.listAssignments( + java.util.Collections.singleton(serverUuid))) { + result.put(assignment.getRoleType(), assignment); + } + return result; + } + + private String adapterUnavailableReason( + String roleType, + PhysicalServerResourceControlAdapterRegistry adapters) { + String error = adapters.getError(roleType); + if (error == null) { + return "RESOURCE_ASSIGNMENT_ADAPTER_MISSING"; + } + int separator = error.indexOf(':'); + return separator < 0 ? error : error.substring(0, separator); + } + + private String unavailableReason( + PhysicalServerResourceControlAdapter adapter, + String serverUuid) { + try { + String reason = adapter.getUnavailableReason(serverUuid); + return reason == null || reason.isEmpty() + ? "RESOURCE_CONSUMER_UNAVAILABLE" : reason; + } catch (RuntimeException error) { + return "RESOURCE_CONSUMER_UNAVAILABLE"; + } + } + + private int isolationOrder(PhysicalServerResourceControlAdapter adapter) { + return adapter != null && adapter.getIsolationMode() + == PhysicalServerResourceIsolationMode.EXCLUSIVE ? 0 : 1; + } + + private Map capacityFootprint( + Collection assignmentValues, + PhysicalServerResourceControlAdapterRegistry adapters) { + Map result = new LinkedHashMap<>(); + assignmentValues.stream() + .sorted(Comparator.comparing( + PhysicalServerResourceAssignmentVO::getRoleType)) + .forEach(assignment -> { + PhysicalServerResourceControlAdapter adapter = + adapters.get(assignment.getRoleType()); + if (adapter == null || adapter.getIsolationMode() + != PhysicalServerResourceIsolationMode.EXCLUSIVE + || assignment.getState() + != PhysicalServerResourceAssignmentState.Synced) { + return; + } + result.put(assignment.getRoleType(), assignment.getCpuSet()); + }); + return result; + } + + private PhysicalServerResourceControlAdapterRegistry adapters() { + return PhysicalServerResourceControlAdapterRegistry.load(pluginRgty); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerRoleType.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerRoleType.java new file mode 100644 index 00000000000..a8e6cc40600 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerRoleType.java @@ -0,0 +1,10 @@ +package org.zstack.physicalserver; + +public final class PhysicalServerRoleType { + public static final String ZBS = "ZBS"; + public static final String MANAGEMENT = "MANAGEMENT"; + public static final String COMPUTE = "COMPUTE"; + + private PhysicalServerRoleType() { + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO.java new file mode 100644 index 00000000000..49de76a03fc --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO.java @@ -0,0 +1,77 @@ +package org.zstack.physicalserver; + +import org.zstack.header.vo.Index; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.PreUpdate; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import java.sql.Timestamp; + +@Entity +@Table(uniqueConstraints = @UniqueConstraint(name = "ukPhysicalServerSerialNumber", columnNames = "serialNumber")) +public class PhysicalServerVO { + @Id + @Column + @Index + private String uuid; + + @Column + private String zoneUuid; + + @Column + private String serialNumber; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + @PreUpdate + private void preUpdate() { + lastOpDate = null; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getZoneUuid() { + return zoneUuid; + } + + public void setZoneUuid(String zoneUuid) { + this.zoneUuid = zoneUuid; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO_.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO_.java new file mode 100644 index 00000000000..10583477ccc --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO_.java @@ -0,0 +1,14 @@ +package org.zstack.physicalserver; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(PhysicalServerVO.class) +public class PhysicalServerVO_ { + public static volatile SingularAttribute uuid; + public static volatile SingularAttribute zoneUuid; + public static volatile SingularAttribute serialNumber; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/RBACInfo.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/RBACInfo.java new file mode 100644 index 00000000000..2f060e28e08 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/RBACInfo.java @@ -0,0 +1,30 @@ +package org.zstack.physicalserver; + +import org.zstack.header.identity.rbac.RBACDescription; + +public class RBACInfo implements RBACDescription { + @Override + public void permissions() { + permissionBuilder() + .name("physicalServer") + .adminOnlyAPIs("org.zstack.physicalserver.**") + .build(); + } + + @Override + public void contributeToRoles() { + } + + @Override + public void roles() { + roleBuilder() + .name("physicalServer") + .uuid("f8d781036ed84b69a9737ec6d78e65e1") + .permissionsByName("physicalServer") + .build(); + } + + @Override + public void globalReadableResources() { + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcileAllPhysicalServersMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcileAllPhysicalServersMsg.java new file mode 100644 index 00000000000..51ef0b3f5a6 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcileAllPhysicalServersMsg.java @@ -0,0 +1,6 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.Message; + +public class ReconcileAllPhysicalServersMsg extends Message { +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcilePhysicalServerMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcilePhysicalServerMsg.java new file mode 100644 index 00000000000..65960a53cd8 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcilePhysicalServerMsg.java @@ -0,0 +1,25 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.Message; + +public class ReconcilePhysicalServerMsg extends Message implements PhysicalServerMessage { + private String serverUuid; + private boolean refreshFacts; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public boolean isRefreshFacts() { + return refreshFacts; + } + + public void setRefreshFacts(boolean refreshFacts) { + this.refreshFacts = refreshFacts; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerResourceAssignmentMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerResourceAssignmentMsg.java new file mode 100644 index 00000000000..ebf7a39d380 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerResourceAssignmentMsg.java @@ -0,0 +1,44 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.NeedReplyMessage; + +public class ReleasePhysicalServerResourceAssignmentMsg extends NeedReplyMessage + implements PhysicalServerMessage { + private String serverUuid; + private String roleType; + private String consumerUuid; + private boolean force; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getRoleType() { + return roleType; + } + + public void setRoleType(String roleType) { + this.roleType = roleType; + } + + public String getConsumerUuid() { + return consumerUuid; + } + + public void setConsumerUuid(String consumerUuid) { + this.consumerUuid = consumerUuid; + } + + public boolean isForce() { + return force; + } + + public void setForce(boolean force) { + this.force = force; + } +} diff --git a/plugin/pom.xml b/plugin/pom.xml index 22f39138fc7..32e060799d3 100755 --- a/plugin/pom.xml +++ b/plugin/pom.xml @@ -33,6 +33,7 @@ directory sshKeyPair hostNetworkInterface + physicalServer vhost expon externalStorage diff --git a/plugin/zbs/pom.xml b/plugin/zbs/pom.xml index c1a8f8e2a85..b94bbd65a43 100644 --- a/plugin/zbs/pom.xml +++ b/plugin/zbs/pom.xml @@ -90,4 +90,4 @@ - \ No newline at end of file + diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/FakeZbsCpuIsolationProvider.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/FakeZbsCpuIsolationProvider.java new file mode 100644 index 00000000000..b90ab36922e --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/FakeZbsCpuIsolationProvider.java @@ -0,0 +1,194 @@ +package org.zstack.storage.zbs; + +import org.zstack.core.CoreGlobalProperty; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_10000; + +public class FakeZbsCpuIsolationProvider implements ZbsCpuIsolationProvider { + private final Map facts = new ConcurrentHashMap<>(); + private final AtomicInteger queryCalls = new AtomicInteger(); + private final AtomicInteger updateCalls = new AtomicInteger(); + private final AtomicBoolean holdNextQuery = new AtomicBoolean(); + private final AtomicBoolean holdNextUpdate = new AtomicBoolean(); + private final AtomicReference heldQuery = new AtomicReference<>(); + private final AtomicReference heldUpdate = new AtomicReference<>(); + private volatile boolean enabled; + private volatile boolean autoApply; + + @Override + public String getProviderType() { + return "FAKE"; + } + + @Override + public boolean isAvailable(ZbsNodeRef nodeRef) { + return CoreGlobalProperty.UNIT_TEST_ON && enabled; + } + + @Override + public void query(ZbsNodeRef nodeRef, ReturnValueCompletion completion) { + queryCalls.incrementAndGet(); + if (!isAvailable(nodeRef)) { + completion.fail(operr(ORG_ZSTACK_CORE_10000, + "FAKE_ZBS_PROVIDER_DISABLED: fake provider never becomes production ready")); + return; + } + ZbsCpuIsolationFact fact = facts.get(nodeRef.getServerUuid()); + if (fact == null) { + completion.fail(operr(ORG_ZSTACK_CORE_10000, + "FAKE_ZBS_FACT_MISSING: no fake fact for physical server[uuid:%s]", nodeRef.getServerUuid())); + return; + } + ZbsCpuIsolationFact response = copy(fact); + if (holdNextQuery.compareAndSet(true, false)) { + heldQuery.set(new HeldQuery(completion, response)); + return; + } + completion.success(response); + } + + @Override + public void update(ZbsNodeRef nodeRef, ZbsCpuIsolationUpdate update, Completion completion) { + updateCalls.incrementAndGet(); + if (!isAvailable(nodeRef)) { + completion.fail(operr(ORG_ZSTACK_CORE_10000, + "FAKE_ZBS_PROVIDER_DISABLED: fake provider never becomes production ready")); + return; + } + if (holdNextUpdate.compareAndSet(true, false)) { + heldUpdate.set(new HeldUpdate(nodeRef, update, completion)); + return; + } + if (autoApply) { + applyUpdate(nodeRef, update); + } + completion.success(); + } + + public void enable() { + enabled = true; + } + + public void disable() { + enabled = false; + autoApply = false; + facts.clear(); + queryCalls.set(0); + updateCalls.set(0); + holdNextQuery.set(false); + holdNextUpdate.set(false); + heldQuery.set(null); + heldUpdate.set(null); + } + + public void setAutoApply(boolean autoApply) { + this.autoApply = autoApply; + } + + public void setFact(String serverUuid, ZbsCpuIsolationFact fact) { + facts.put(serverUuid, copy(fact)); + } + + public int getQueryCalls() { + return queryCalls.get(); + } + + public int getUpdateCalls() { + return updateCalls.get(); + } + + public void holdNextQuery() { + holdNextQuery.set(true); + } + + public boolean hasHeldQuery() { + return heldQuery.get() != null; + } + + public void completeHeldQuery() { + HeldQuery held = heldQuery.getAndSet(null); + if (held != null) { + held.completion.success(held.fact); + } + } + + public void holdNextUpdate() { + holdNextUpdate.set(true); + } + + public boolean hasHeldUpdate() { + return heldUpdate.get() != null; + } + + public void completeHeldUpdate() { + HeldUpdate held = heldUpdate.getAndSet(null); + if (held != null) { + if (autoApply) { + applyUpdate(held.nodeRef, held.update); + } + held.completion.success(); + } + } + + private void applyUpdate(ZbsNodeRef nodeRef, ZbsCpuIsolationUpdate update) { + ZbsCpuIsolationFact fact = new ZbsCpuIsolationFact(); + fact.setIsolationState( + "DISABLE".equals(update.getOperation()) ? "DISABLED" : "READY"); + fact.setConfiguredCpuSet(update.getDesiredCpuSet()); + fact.setEffectiveCpuSet(update.getDesiredCpuSet()); + fact.setMemory(update.getMemory() != null && update.getMemory() == 0L + ? null : update.getMemory()); + fact.setExpectedServiceCount(1); + fact.setCoveredServiceCount(1); + fact.setServiceReady(true); + facts.put(nodeRef.getServerUuid(), fact); + } + + private ZbsCpuIsolationFact copy(ZbsCpuIsolationFact source) { + ZbsCpuIsolationFact copy = new ZbsCpuIsolationFact(); + copy.setIsolationState(source.getIsolationState()); + copy.setConfiguredCpuSet(source.getConfiguredCpuSet()); + copy.setEffectiveCpuSet(source.getEffectiveCpuSet()); + copy.setMemory(source.getMemory()); + copy.setCoveredServiceCount(source.getCoveredServiceCount()); + copy.setExpectedServiceCount(source.getExpectedServiceCount()); + copy.setServiceReady(source.isServiceReady()); + return copy; + } + + private static class HeldQuery { + private final ReturnValueCompletion completion; + private final ZbsCpuIsolationFact fact; + + private HeldQuery( + ReturnValueCompletion completion, + ZbsCpuIsolationFact fact) { + this.completion = completion; + this.fact = fact; + } + } + + private static class HeldUpdate { + private final ZbsNodeRef nodeRef; + private final ZbsCpuIsolationUpdate update; + private final Completion completion; + + private HeldUpdate( + ZbsNodeRef nodeRef, + ZbsCpuIsolationUpdate update, + Completion completion) { + this.nodeRef = nodeRef; + this.update = update; + this.completion = completion; + } + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/MdsInfo.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/MdsInfo.java index 4085e6657af..1d6b914a235 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/MdsInfo.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/MdsInfo.java @@ -15,6 +15,7 @@ public class MdsInfo { private int port = 22; private String addr; private String externalAddr; + private String physicalServerSerialNumber; private MdsStatus status; @Override @@ -74,6 +75,14 @@ public void setExternalAddr(String externalAddr) { this.externalAddr = externalAddr; } + public String getPhysicalServerSerialNumber() { + return physicalServerSerialNumber; + } + + public void setPhysicalServerSerialNumber(String physicalServerSerialNumber) { + this.physicalServerSerialNumber = physicalServerSerialNumber; + } + public MdsStatus getStatus() { return status; } diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationCascadeExtension.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationCascadeExtension.java new file mode 100644 index 00000000000..6241299a6d9 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationCascadeExtension.java @@ -0,0 +1,244 @@ +package org.zstack.storage.zbs; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cascade.AbstractAsyncCascadeExtension; +import org.zstack.core.cascade.CascadeAction; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.core.cloudbus.EventCallback; +import org.zstack.core.cloudbus.EventFacade; +import org.zstack.core.db.Q; +import org.zstack.header.Component; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_; +import org.zstack.header.storage.primary.PrimaryStorageCanonicalEvent; +import org.zstack.header.storage.primary.PrimaryStorageInventory; +import org.zstack.header.storage.primary.PrimaryStorageVO; +import org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_10000; + +public class ZbsCpuIsolationCascadeExtension extends AbstractAsyncCascadeExtension implements Component { + private static final String NAME = "ZbsCpuIsolationAssignment"; + private static final CLogger logger = Utils.getLogger( + ZbsCpuIsolationCascadeExtension.class); + + @Autowired + private EventFacade evtf; + @Autowired + private ZbsNodeRefContributor nodeRefs; + @Autowired + private ZbsPhysicalServerIdentityResolver physicalServerIdentities; + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + + private EventCallback addonInfoChanged; + private EventCallback primaryStorageDeleted; + + @Override + public void asyncCascade(CascadeAction action, Completion completion) { + if (!PrimaryStorageVO.class.getSimpleName().equals(action.getParentIssuer()) + || !CascadeConstant.DELETION_CODES.contains(action.getActionCode())) { + completion.success(); + return; + } + + Set deleting = deletingZbsPrimaryStorages(action); + if (deleting.isEmpty()) { + completion.success(); + return; + } + + List lastRelationServers = lastRelationServers(deleting); + if (action.isActionCode(CascadeConstant.DELETION_CHECK_CODE) + || lastRelationServers.isEmpty()) { + completion.success(); + return; + } + + release( + lastRelationServers, + action.isActionCode(CascadeConstant.DELETION_FORCE_DELETE_CODE), + completion); + } + + public void beforePersistAddonInfo( + String primaryStorageUuid, + AddonInfo addonInfo, + Completion completion) { + Set newRelationServers = + physicalServerIdentities.resolveServerUuids( + primaryStorageUuid, addonInfo); + String oldAddonInfo = Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, primaryStorageUuid) + .findValue(); + if (oldAddonInfo == null || oldAddonInfo.isEmpty()) { + completion.success(); + return; + } + List removedLastRelations = lastRelationServers( + Collections.singleton(primaryStorageUuid)); + removedLastRelations.removeAll(newRelationServers); + release(removedLastRelations, false, completion); + } + + private void release( + List serverUuids, + boolean force, + Completion completion) { + if (serverUuids.isEmpty()) { + completion.success(); + return; + } + new While<>(serverUuids).each((serverUuid, each) -> { + if (physicalServerManager == null) { + each.done(); + return; + } + physicalServerManager.releaseResourceAssignment( + serverUuid, + ZbsResourceAssignmentBackend.ROLE_TYPE, + null, + force, + new Completion(each) { + @Override + public void success() { + each.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + if (force) { + logger.error(String.format( + "failed to release ZBS resource assignment before force deleting " + + "the last relation for physical server[uuid:%s]: %s", + serverUuid, errorCode)); + each.done(); + return; + } + each.addError(errorCode); + each.allDone(); + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + completion.fail(errorCodeList.getCauses().get(0)); + return; + } + completion.success(); + } + }); + } + + private Set deletingZbsPrimaryStorages(CascadeAction action) { + List inventories = action.getParentIssuerContext(); + if (inventories == null || inventories.isEmpty()) { + return Collections.emptySet(); + } + List candidates = new ArrayList<>(); + for (PrimaryStorageInventory inventory : inventories) { + candidates.add(inventory.getUuid()); + } + return new HashSet<>(Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.uuid) + .in(ExternalPrimaryStorageVO_.uuid, candidates) + .eq(ExternalPrimaryStorageVO_.identity, ZbsConstants.IDENTITY) + .listValues()); + } + + private List lastRelationServers(Set deletingPrimaryStorageUuids) { + Map refs = nodeRefs.bulkList(Collections.emptySet()); + List result = new ArrayList<>(); + for (ZbsNodeRef ref : refs.values()) { + if (!ref.includesAnyPrimaryStorage(deletingPrimaryStorageUuids)) { + continue; + } + if (ref.getReasonCode() != null) { + throw new OperationFailureException(operr( + ORG_ZSTACK_CORE_10000, + "cannot determine the last ZBS relationship for physical server[uuid:%s], reason[%s]", + ref.getServerUuid(), ref.getReasonCode())); + } + boolean hasRemainingRelationship = ref.getPrimaryStorageUuids().stream() + .anyMatch(uuid -> !deletingPrimaryStorageUuids.contains(uuid)); + if (!hasRemainingRelationship) { + result.add(ref.getServerUuid()); + } + } + return result; + } + + private void sendReconcileAll() { + if (physicalServerManager != null) { + physicalServerManager.reconcileAll(); + } + } + + @Override + public boolean start() { + addonInfoChanged = new EventCallback() { + @Override + protected void run( + Map tokens, + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data) { + sendReconcileAll(); + } + }; + primaryStorageDeleted = new EventCallback() { + @Override + protected void run( + Map tokens, + PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData data) { + sendReconcileAll(); + } + }; + evtf.on(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, addonInfoChanged); + evtf.on(PrimaryStorageCanonicalEvent.PRIMARY_STORAGE_DELETED_PATH, primaryStorageDeleted); + return true; + } + + @Override + public boolean stop() { + if (addonInfoChanged != null) { + evtf.off(addonInfoChanged); + } + if (primaryStorageDeleted != null) { + evtf.off(primaryStorageDeleted); + } + return true; + } + + @Override + public List getEdgeNames() { + return Collections.singletonList(PrimaryStorageVO.class.getSimpleName()); + } + + @Override + public String getCascadeResourceName() { + return NAME; + } + + @Override + public CascadeAction createActionForChildResource(CascadeAction action) { + return null; + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationFact.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationFact.java new file mode 100644 index 00000000000..3debcc88a91 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationFact.java @@ -0,0 +1,68 @@ +package org.zstack.storage.zbs; + +public class ZbsCpuIsolationFact { + private String isolationState; + private String configuredCpuSet; + private String effectiveCpuSet; + private Long memory; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private boolean serviceReady; + + public String getIsolationState() { + return isolationState; + } + + public void setIsolationState(String isolationState) { + this.isolationState = isolationState; + } + + public String getConfiguredCpuSet() { + return configuredCpuSet; + } + + public void setConfiguredCpuSet(String configuredCpuSet) { + this.configuredCpuSet = configuredCpuSet; + } + + public String getEffectiveCpuSet() { + return effectiveCpuSet; + } + + public void setEffectiveCpuSet(String effectiveCpuSet) { + this.effectiveCpuSet = effectiveCpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public boolean isServiceReady() { + return serviceReady; + } + + public void setServiceReady(boolean serviceReady) { + this.serviceReady = serviceReady; + } + +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationProvider.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationProvider.java new file mode 100644 index 00000000000..d522665604a --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationProvider.java @@ -0,0 +1,14 @@ +package org.zstack.storage.zbs; + +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; + +public interface ZbsCpuIsolationProvider { + String getProviderType(); + + boolean isAvailable(ZbsNodeRef nodeRef); + + void query(ZbsNodeRef nodeRef, ReturnValueCompletion completion); + + void update(ZbsNodeRef nodeRef, ZbsCpuIsolationUpdate update, Completion completion); +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationUpdate.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationUpdate.java new file mode 100644 index 00000000000..f33f8f03f0d --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationUpdate.java @@ -0,0 +1,31 @@ +package org.zstack.storage.zbs; + +public class ZbsCpuIsolationUpdate { + private String operation; + private String desiredCpuSet; + private Long memory; + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getDesiredCpuSet() { + return desiredCpuSet; + } + + public void setDesiredCpuSet(String desiredCpuSet) { + this.desiredCpuSet = desiredCpuSet; + } + + public Long getMemory() { + return memory; + } + + public void setMemory(Long memory) { + this.memory = memory; + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRef.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRef.java new file mode 100644 index 00000000000..5ba48326826 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRef.java @@ -0,0 +1,64 @@ +package org.zstack.storage.zbs; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +public class ZbsNodeRef { + private String serverUuid; + private final Set primaryStorageUuids = new LinkedHashSet<>(); + private final Set nodeAddresses = new LinkedHashSet<>(); + private int sourceRefCount; + private String reasonCode; + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public List getPrimaryStorageUuids() { + return new ArrayList<>(primaryStorageUuids); + } + + public void addPrimaryStorageUuid(String primaryStorageUuid) { + primaryStorageUuids.add(primaryStorageUuid); + } + + public List getNodeAddresses() { + return new ArrayList<>(nodeAddresses); + } + + public void addNodeAddress(String nodeAddress) { + nodeAddresses.add(nodeAddress); + } + + public int getSourceRefCount() { + return sourceRefCount; + } + + public void incrementSourceRefCount() { + sourceRefCount++; + } + + public String getReasonCode() { + return reasonCode; + } + + public void setReasonCode(String reasonCode) { + this.reasonCode = reasonCode; + } + + public boolean includesAnyPrimaryStorage(Collection uuids) { + for (String uuid : uuids) { + if (primaryStorageUuids.contains(uuid)) { + return true; + } + } + return false; + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributor.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributor.java new file mode 100644 index 00000000000..4d6370799f9 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributor.java @@ -0,0 +1,8 @@ +package org.zstack.storage.zbs; + +import java.util.Collection; +import java.util.Map; + +public interface ZbsNodeRefContributor { + Map bulkList(Collection serverUuids); +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributorImpl.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributorImpl.java new file mode 100644 index 00000000000..1e6ad48da9f --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributorImpl.java @@ -0,0 +1,333 @@ +package org.zstack.storage.zbs; + +import org.zstack.compute.host.HostSystemTags; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.Platform; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostVO_; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_; +import org.zstack.header.tag.SystemTagVO; +import org.zstack.header.tag.SystemTagVO_; +import org.zstack.kvm.KVMHostVO; +import org.zstack.utils.TagUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.gson.JSONObjectUtil; +import org.zstack.utils.logging.CLogger; + +import java.util.Collection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_10000; + +public class ZbsNodeRefContributorImpl implements ZbsNodeRefContributor, + ZbsPhysicalServerIdentityResolver { + private static final CLogger logger = Utils.getLogger( + ZbsNodeRefContributorImpl.class); + + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + + @Override + public Map bulkList(Collection serverUuids) { + Set requestedServerUuids = serverUuids == null + ? Collections.emptySet() : new HashSet<>(serverUuids); + List primaryStorages = Q.New(ExternalPrimaryStorageVO.class) + .eq(ExternalPrimaryStorageVO_.identity, ZbsConstants.IDENTITY) + .list(); + Map addonInfos = new LinkedHashMap<>(); + for (ExternalPrimaryStorageVO primaryStorage : primaryStorages) { + addonInfos.put(primaryStorage.getUuid(), parseAddonInfo(primaryStorage)); + } + Map serversBySerialNumber = serversBySerialNumber( + addonInfos.values()); + Map> serversByIp = requiresIpFallback( + addonInfos.values()) + ? mapServersByIp(listHosts(requestedServerUuids)) + : Collections.emptyMap(); + Map result = new LinkedHashMap<>(); + for (Map.Entry source : addonInfos.entrySet()) { + for (MdsInfo mds : source.getValue().getMdsInfos()) { + if (mds == null) { + throw invalidAddonInfo(source.getKey(), + "contains an empty mdsInfo"); + } + String serialNumber = serialNumber(mds); + if (serialNumber != null) { + String serverUuid = serversBySerialNumber.get(serialNumber); + if (serverUuid == null) { + logger.warn(String.format( + "cannot resolve ZBS MDS physical server serialNumber[%s], " + + "reasonCode[ZBS_NODE_SERIAL_UNRESOLVED]", + serialNumber)); + continue; + } + if (requestedServerUuids.isEmpty() + || requestedServerUuids.contains(serverUuid)) { + addRef(result, serverUuid, source.getKey(), + nodeAddress(mds), null); + } + continue; + } + String address = resolveAddress(mds, serversByIp); + if (address == null) { + continue; + } + Set matchedServers = serversByIp.get(address); + for (String serverUuid : matchedServers) { + addRef(result, serverUuid, source.getKey(), address, + matchedServers.size() > 1 + ? "ZBS_NODE_ADDRESS_AMBIGUOUS" + : "ZBS_NODE_SERIAL_MISSING"); + } + } + } + return result; + } + + @Override + public Set resolveServerUuids( + String primaryStorageUuid, AddonInfo addonInfo) { + validateAddonInfo(primaryStorageUuid, addonInfo); + Map serversBySerialNumber = serversBySerialNumber( + Collections.singleton(addonInfo)); + Map> serversByIp = requiresIpFallback( + Collections.singleton(addonInfo)) + ? mapServersByIp(listHosts(Collections.emptySet())) + : Collections.emptyMap(); + Set result = new HashSet<>(); + for (MdsInfo mds : addonInfo.getMdsInfos()) { + if (mds == null) { + throw invalidAddonInfo(primaryStorageUuid, "contains an empty mdsInfo"); + } + String serialNumber = serialNumber(mds); + if (serialNumber != null) { + String serverUuid = serversBySerialNumber.get(serialNumber); + if (serverUuid != null) { + result.add(serverUuid); + } + continue; + } + String address = resolveAddress(mds, serversByIp); + if (address != null) { + result.addAll(serversByIp.get(address)); + } + } + return result; + } + + @Override + public void enrichPhysicalServerSerialNumbers( + String primaryStorageUuid, AddonInfo addonInfo) { + validateAddonInfo(primaryStorageUuid, addonInfo); + if (!requiresIpFallback(Collections.singleton(addonInfo))) { + normalizeSerialNumbers(addonInfo); + return; + } + Map> serversByIp = mapServersByIp( + listHosts(Collections.emptySet())); + Map matchedServers = new LinkedHashMap<>(); + for (MdsInfo mds : addonInfo.getMdsInfos()) { + if (mds == null) { + throw invalidAddonInfo(primaryStorageUuid, + "contains an empty mdsInfo"); + } + String serialNumber = serialNumber(mds); + if (serialNumber != null) { + mds.setPhysicalServerSerialNumber(serialNumber); + continue; + } + String address = resolveAddress(mds, serversByIp); + if (address == null || serversByIp.get(address).size() != 1) { + continue; + } + matchedServers.put(mds, serversByIp.get(address).iterator().next()); + } + Map serialNumbersByServer = serialNumbersByServer( + new HashSet<>(matchedServers.values())); + for (Map.Entry match : matchedServers.entrySet()) { + match.getKey().setPhysicalServerSerialNumber( + serialNumbersByServer.get(match.getValue())); + } + } + + private List listHosts(Collection serverUuids) { + Q hostQuery = Q.New(KVMHostVO.class) + .notNull(HostVO_.serverUuid); + if (serverUuids != null && !serverUuids.isEmpty()) { + hostQuery.in(HostVO_.serverUuid, serverUuids); + } + return hostQuery.list(); + } + + private AddonInfo parseAddonInfo(ExternalPrimaryStorageVO primaryStorage) { + if (primaryStorage.getAddonInfo() == null || primaryStorage.getAddonInfo().isEmpty()) { + throw invalidAddonInfo(primaryStorage.getUuid(), "is empty"); + } + + AddonInfo addonInfo = JSONObjectUtil.toObject( + primaryStorage.getAddonInfo(), AddonInfo.class); + validateAddonInfo(primaryStorage.getUuid(), addonInfo); + return addonInfo; + } + + private void validateAddonInfo(String primaryStorageUuid, AddonInfo addonInfo) { + if (addonInfo == null || addonInfo.getMdsInfos() == null + || addonInfo.getMdsInfos().isEmpty()) { + throw invalidAddonInfo(primaryStorageUuid, "does not contain mdsInfos"); + } + } + + private OperationFailureException invalidAddonInfo(String primaryStorageUuid, String detail) { + return new OperationFailureException(operr( + ORG_ZSTACK_CORE_10000, + "ZBS_NODE_RELATION_SOURCE_INVALID: cannot derive ZBS node relations because " + + "primary storage[uuid:%s] addonInfo %s", + primaryStorageUuid, + detail)); + } + + private Map serversBySerialNumber( + Collection addonInfos) { + Set serialNumbers = new HashSet<>(); + for (AddonInfo addonInfo : addonInfos) { + for (MdsInfo mds : addonInfo.getMdsInfos()) { + if (mds == null) { + continue; + } + String serialNumber = serialNumber(mds); + if (serialNumber != null) { + serialNumbers.add(serialNumber); + } + } + } + if (serialNumbers.isEmpty()) { + return Collections.emptyMap(); + } + if (physicalServerManager == null) { + return Collections.emptyMap(); + } + List identities = new ArrayList<>(); + for (String serialNumber : serialNumbers) { + identities.add(new PhysicalServerIdentitySpec(serialNumber, null)); + } + return physicalServerManager.resolveIdentities(identities); + } + + private boolean requiresIpFallback(Collection addonInfos) { + for (AddonInfo addonInfo : addonInfos) { + for (MdsInfo mds : addonInfo.getMdsInfos()) { + if (mds == null || serialNumber(mds) == null) { + return true; + } + } + } + return false; + } + + private void normalizeSerialNumbers(AddonInfo addonInfo) { + for (MdsInfo mds : addonInfo.getMdsInfos()) { + if (mds != null) { + mds.setPhysicalServerSerialNumber(serialNumber(mds)); + } + } + } + + private Map serialNumbersByServer( + Collection serverUuids) { + if (serverUuids.isEmpty()) { + return Collections.emptyMap(); + } + return physicalServerManager == null + ? Collections.emptyMap() + : physicalServerManager.findSerialNumbersByServerUuids(serverUuids); + } + + private String serialNumber(MdsInfo mds) { + return Platform.normalizeMachineSerialNumber( + mds.getPhysicalServerSerialNumber()); + } + + private String nodeAddress(MdsInfo mds) { + return mds.getExternalAddr() == null ? mds.getAddr() : mds.getExternalAddr(); + } + + private void addRef( + Map refs, + String serverUuid, + String primaryStorageUuid, + String nodeAddress, + String reasonCode) { + ZbsNodeRef ref = refs.computeIfAbsent(serverUuid, ignored -> { + ZbsNodeRef created = new ZbsNodeRef(); + created.setServerUuid(serverUuid); + return created; + }); + ref.addPrimaryStorageUuid(primaryStorageUuid); + if (nodeAddress != null) { + ref.addNodeAddress(nodeAddress); + } + ref.incrementSourceRefCount(); + if (reasonCode != null) { + ref.setReasonCode(reasonCode); + } + } + + private Map> mapServersByIp(List hosts) { + Map> result = new HashMap<>(); + if (hosts.isEmpty()) { + return result; + } + Map serverByHost = new HashMap<>(); + for (KVMHostVO host : hosts) { + serverByHost.put(host.getUuid(), host.getServerUuid()); + add(result, host.getManagementIp(), host.getServerUuid()); + } + + List tags = Q.New(SystemTagVO.class) + .in(SystemTagVO_.resourceUuid, serverByHost.keySet()) + .eq(SystemTagVO_.resourceType, org.zstack.header.host.HostVO.class.getSimpleName()) + .like(SystemTagVO_.tag, + TagUtils.tagPatternToSqlPattern(HostSystemTags.EXTRA_IPS.getTagFormat())) + .list(); + for (SystemTagVO tag : tags) { + String extraIps = HostSystemTags.EXTRA_IPS.getTokenByTag( + tag.getTag(), HostSystemTags.EXTRA_IPS_TOKEN); + if (extraIps == null) { + continue; + } + for (String ip : extraIps.split(",")) { + add(result, ip.trim(), serverByHost.get(tag.getResourceUuid())); + } + } + return result; + } + + private String resolveAddress(MdsInfo mds, Map> serversByIp) { + if (mds.getExternalAddr() != null && serversByIp.containsKey(mds.getExternalAddr())) { + return mds.getExternalAddr(); + } + if (mds.getAddr() != null && serversByIp.containsKey(mds.getAddr())) { + return mds.getAddr(); + } + return null; + } + + private void add(Map> values, String key, String value) { + if (key == null || key.isEmpty() || value == null) { + return; + } + values.computeIfAbsent(key, ignored -> new HashSet<>()).add(value); + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPhysicalServerIdentityResolver.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPhysicalServerIdentityResolver.java new file mode 100644 index 00000000000..48c52027ad5 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPhysicalServerIdentityResolver.java @@ -0,0 +1,11 @@ +package org.zstack.storage.zbs; + +import java.util.Set; + +public interface ZbsPhysicalServerIdentityResolver { + Set resolveServerUuids( + String primaryStorageUuid, AddonInfo addonInfo); + + void enrichPhysicalServerSerialNumbers( + String primaryStorageUuid, AddonInfo addonInfo); +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java index 416cc620572..32ace2c86b5 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java @@ -219,6 +219,11 @@ public void success(SyncMetadataRsp ret) { } getSelf().setExternalAddr(ret.getExternalAddr()); + String serialNumber = Platform.normalizeMachineSerialNumber( + ret.getPhysicalServerSerialNumber()); + if (serialNumber != null) { + getSelf().setPhysicalServerSerialNumber(serialNumber); + } trigger.next(); } @@ -413,6 +418,7 @@ public void setClusterInfo(ClusterInfo clusterInfo) { public static class SyncMetadataRsp extends ZbsMdsBase.AgentResponse { private String externalAddr; + private String physicalServerSerialNumber; public String getExternalAddr() { return externalAddr; @@ -421,6 +427,15 @@ public String getExternalAddr() { public void setExternalAddr(String externalAddr) { this.externalAddr = externalAddr; } + + public String getPhysicalServerSerialNumber() { + return physicalServerSerialNumber; + } + + public void setPhysicalServerSerialNumber( + String physicalServerSerialNumber) { + this.physicalServerSerialNumber = physicalServerSerialNumber; + } } public static class SyncMetadataCmd extends ZbsMdsBase.AgentCommand { diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsResourceAssignmentBackend.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsResourceAssignmentBackend.java new file mode 100644 index 00000000000..68327c92b49 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsResourceAssignmentBackend.java @@ -0,0 +1,566 @@ +package org.zstack.storage.zbs; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.core.thread.ThreadFacadeImpl; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerResourceApplicationMode; +import org.zstack.header.physicalserver.PhysicalServerResourceConsumerState; +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter; +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode; +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; +import org.zstack.header.physicalserver.ResourceControlCommand; +import org.zstack.header.physicalserver.ResourceControlResponse; +import org.zstack.header.physicalserver.RoleServiceManifest; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_10000; + +public class ZbsResourceAssignmentBackend implements + PhysicalServerResourceControlAdapter { + public static final String ROLE_TYPE = "ZBS"; + public static final String ROLE_SERVICE_MANIFEST_PATH = + "physical-server-roles/zbs.yaml"; + private static final String TOPOLOGY_ROLE_TYPE = "COMPUTE"; + private static final CLogger logger = Utils.getLogger( + ZbsResourceAssignmentBackend.class); + private static final RoleServiceManifest ROLE_SERVICES = + RoleServiceManifest.load( + ROLE_SERVICE_MANIFEST_PATH, + ROLE_TYPE, + PhysicalServerResourceApplicationMode.PROVIDER_MANAGED); + + private final AtomicReference> zbsRefs = + new AtomicReference<>(Collections.emptyMap()); + + @Autowired + private PluginRegistry pluginRgty; + @Autowired + private ThreadFacade thdf; + + @Override + public String getRoleType() { + return ROLE_SERVICES.getRoleType(); + } + + @Override + public PhysicalServerResourceIsolationMode getIsolationMode() { + return PhysicalServerResourceIsolationMode.EXCLUSIVE; + } + + @Override + public PhysicalServerResourceApplicationMode getApplicationMode() { + return PhysicalServerResourceApplicationMode.PROVIDER_MANAGED; + } + + @Override + public String getTopologyRoleType() { + return TOPOLOGY_ROLE_TYPE; + } + + @Override + public String getDefaultCpuSet( + PhysicalServerCpuTopology topology, + Set allocatedExclusiveCpus) { + List groups = + topology.getCoreGroups(); + int selectedCount = groups.size() / 2; + if (selectedCount < 1) { + throw new IllegalArgumentException( + "CPU_TOPOLOGY_TOO_SMALL: default ZBS isolation selects no complete core group"); + } + SortedSet selected = new TreeSet<>(); + for (int index = groups.size() - selectedCount; + index < groups.size(); index++) { + selected.addAll(groups.get(index).getCpus()); + } + return PhysicalServerCpuSet.format(selected); + } + + @Override + public void refreshAssociations() { + refreshRefs(Collections.emptySet()); + } + + @Override + public void refreshAssociations(Collection serverUuids) { + refreshRefs(serverUuids); + } + + @Override + public Set getAssociatedServerUuids() { + return new HashSet<>(zbsRefs.get().keySet()); + } + + @Override + public Set getEligibleDefaultServerUuids() { + return getAssociatedServerUuids(); + } + + @Override + public PhysicalServerResourceConsumerState getState(String serverUuid) { + ZbsNodeRef ref = zbsRefs.get().get(serverUuid); + if (ref == null) { + return PhysicalServerResourceConsumerState.MISSING; + } + if (ref.getReasonCode() != null + || resolveProvider(ref).provider == null) { + return PhysicalServerResourceConsumerState.UNAVAILABLE; + } + return PhysicalServerResourceConsumerState.AVAILABLE; + } + + @Override + public String getUnavailableReason(String serverUuid) { + ZbsNodeRef ref = zbsRefs.get().get(serverUuid); + if (ref == null) { + return "ZBS_NODE_RELATION_MISSING"; + } + return resolveProvider(ref).reasonCode; + } + + @Override + public Map getStates( + Collection serverUuids) { + Map result = + new HashMap<>(); + if (serverUuids == null) { + return result; + } + for (String serverUuid : serverUuids) { + result.put(serverUuid, getState(serverUuid)); + } + return result; + } + + @Override + public void collectTopology( + String serverUuid, + ReturnValueCompletion completion) { + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "ZBS uses topologyRoleType[%s]", TOPOLOGY_ROLE_TYPE)); + } + + @Override + public void apply( + String serverUuid, + String consumerUuid, + ResourceControlCommand command, + ReturnValueCompletion completion) { + ZbsNodeRef ref = zbsRefs.get().get(serverUuid); + ProviderResolution resolution = resolveProvider(ref); + if (resolution.provider == null) { + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "%s: physical server[uuid:%s]", + resolution.reasonCode, serverUuid)); + return; + } + queryProvider( + resolution.provider, + ref, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + ResourceControlResponse observed; + try { + observed = toResponse(fact); + } catch (RuntimeException error) { + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "%s", error.getMessage())); + return; + } + if (!shouldUpdate(command, fact)) { + completion.success(observed); + return; + } + ZbsCpuIsolationUpdate update = update(command); + updateProvider( + resolution.provider, + ref, + update, + new Completion(completion) { + @Override + public void success() { + queryProvider( + resolution.provider, + ref, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + completeFact(fact, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void collectManagedServiceUsage( + String serverUuid, + boolean includeAuxiliaryServices, + ReturnValueCompletion> completion) { + ZbsNodeRef ref = zbsRefs.get().get(serverUuid); + ProviderResolution resolution = resolveProvider(ref); + if (resolution.provider == null) { + completion.success(ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE")); + return; + } + queryProvider( + resolution.provider, + ref, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + ManagedServiceResourceUsage usage = + new ManagedServiceResourceUsage(); + usage.setRoleType(ROLE_TYPE); + usage.setServiceName( + ROLE_SERVICES.getServices().get(0).getName()); + usage.setRestartable(false); + usage.setState(fact.isServiceReady() + ? "RUNNING" : "UNAVAILABLE"); + usage.setCpuSet(normalize(fact.getEffectiveCpuSet())); + usage.setMemoryLimit(fact.getMemory()); + completion.success(Collections.singletonList(usage)); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to query ZBS managed service on physical server[uuid:%s]: %s", + serverUuid, errorCode)); + completion.success(ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE")); + } + }); + } + + @Override + public void restartManagedServices( + String serverUuid, + boolean includeAuxiliaryServices, + Collection serviceNames, + Completion completion) { + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "SERVICE_RESTART_NOT_SUPPORTED: roleType[%s] is provider-managed", + ROLE_TYPE)); + } + + private boolean shouldUpdate( + ResourceControlCommand command, + ZbsCpuIsolationFact fact) { + boolean release = "RELEASE".equals(command.getOperation()); + if (release) { + return fact == null + || !"DISABLED".equals(fact.getIsolationState()) + || !empty(fact.getEffectiveCpuSet()) + || fact.getMemory() != null && fact.getMemory() != 0L; + } + if (fact == null || "UNSUPPORTED".equals(fact.getIsolationState())) { + return false; + } + return !"READY".equals(fact.getIsolationState()) + || !normalize(command.getCpuSet()).equals( + normalize(fact.getEffectiveCpuSet())) + || !memoryMatches(command.getMemory(), fact.getMemory()); + } + + private ZbsCpuIsolationUpdate update(ResourceControlCommand command) { + boolean release = "RELEASE".equals(command.getOperation()); + ZbsCpuIsolationUpdate update = new ZbsCpuIsolationUpdate(); + update.setOperation(release ? "DISABLE" : "SET"); + update.setDesiredCpuSet(release ? "" : command.getCpuSet()); + update.setMemory(release ? Long.valueOf(0L) : command.getMemory()); + return update; + } + + private void completeFact( + ZbsCpuIsolationFact fact, + ReturnValueCompletion completion) { + try { + completion.success(toResponse(fact)); + } catch (RuntimeException error) { + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "%s", error.getMessage())); + } + } + + private ResourceControlResponse toResponse(ZbsCpuIsolationFact fact) { + if (fact == null) { + throw new IllegalArgumentException( + "OWNER_FACT_INVALID: Provider returned null"); + } + if (!isOwnerState(fact.getIsolationState())) { + throw new IllegalArgumentException( + "OWNER_FACT_INVALID: isolationState is not supported"); + } + if ("READY".equals(fact.getIsolationState()) + && (!fact.isServiceReady() + || empty(fact.getEffectiveCpuSet()) + || !normalize(fact.getConfiguredCpuSet()).equals( + normalize(fact.getEffectiveCpuSet())) + || fact.getExpectedServiceCount() == null + || fact.getExpectedServiceCount() < 1 + || !fact.getExpectedServiceCount().equals( + fact.getCoveredServiceCount()))) { + throw new IllegalArgumentException( + "OWNER_SERVICE_NOT_READY: READY requires identical CPU sets and complete service coverage"); + } + if ("DISABLED".equals(fact.getIsolationState()) + && !empty(fact.getEffectiveCpuSet())) { + throw new IllegalArgumentException( + "OWNER_FACT_INVALID: DISABLED must not report an effective CPUSet"); + } + + ResourceControlResponse response = new ResourceControlResponse(); + response.setState(fact.getIsolationState()); + response.setCpuSet(normalize(fact.getEffectiveCpuSet())); + response.setMemory(fact.getMemory()); + response.setCoveredServiceCount(fact.getCoveredServiceCount()); + response.setExpectedServiceCount(fact.getExpectedServiceCount()); + return response; + } + + private void refreshRefs(Collection serverUuids) { + List contributors = + pluginRgty.getExtensionList(ZbsNodeRefContributor.class); + Map loaded = new HashMap<>(); + for (ZbsNodeRefContributor contributor : contributors) { + Map contribution; + try { + contribution = contributor.bulkList(serverUuids); + } catch (RuntimeException error) { + logger.warn(String.format( + "failed to refresh ZBS node relations from contributor[%s]: %s", + contributor.getClass().getName(), error.getMessage())); + return; + } + for (Map.Entry entry : contribution.entrySet()) { + if (loaded.put(entry.getKey(), entry.getValue()) != null) { + entry.getValue().setReasonCode( + "ZBS_NODE_REF_CONTRIBUTOR_AMBIGUOUS"); + } + } + } + if (serverUuids == null || serverUuids.isEmpty()) { + zbsRefs.set(Collections.unmodifiableMap(loaded)); + return; + } + while (true) { + Map current = zbsRefs.get(); + Map replacement = new HashMap<>(current); + for (String serverUuid : serverUuids) { + replacement.remove(serverUuid); + } + replacement.putAll(loaded); + if (zbsRefs.compareAndSet( + current, Collections.unmodifiableMap(replacement))) { + return; + } + } + } + + private ProviderResolution resolveProvider(ZbsNodeRef ref) { + if (ref == null) { + return new ProviderResolution(null, "ZBS_NODE_RELATION_MISSING"); + } + if (ref.getReasonCode() != null) { + return new ProviderResolution(null, ref.getReasonCode()); + } + List available = new ArrayList<>(); + for (ZbsCpuIsolationProvider provider : + pluginRgty.getExtensionList(ZbsCpuIsolationProvider.class)) { + if (provider.isAvailable(ref)) { + available.add(provider); + } + } + if (available.isEmpty()) { + return new ProviderResolution(null, "ZBS_PROVIDER_UNAVAILABLE"); + } + if (available.size() > 1) { + return new ProviderResolution(null, "ZBS_PROVIDER_AMBIGUOUS"); + } + return new ProviderResolution(available.get(0), null); + } + + private void queryProvider( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + ReturnValueCompletion completion) { + long timeout = providerTimeoutMillis(); + AtomicBoolean completed = new AtomicBoolean(); + ThreadFacadeImpl.TimeoutTaskReceipt receipt = thdf.submitTimeoutTask( + () -> { + if (completed.compareAndSet(false, true)) { + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "ZBS_PROVIDER_CALL_TIMEOUT: Provider[%s] Query timed out", + provider.getProviderType())); + } + }, + TimeUnit.MILLISECONDS, + timeout); + try { + provider.query( + ref, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.success(fact); + } + } + + @Override + public void fail(ErrorCode errorCode) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.fail(errorCode); + } + } + }); + } catch (RuntimeException error) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "ZBS_PROVIDER_CALL_FAILED: Provider[%s] Query failed: %s", + provider.getProviderType(), error.getMessage())); + } + } + } + + private void updateProvider( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + ZbsCpuIsolationUpdate update, + Completion completion) { + long timeout = providerTimeoutMillis(); + AtomicBoolean completed = new AtomicBoolean(); + ThreadFacadeImpl.TimeoutTaskReceipt receipt = thdf.submitTimeoutTask( + () -> { + if (completed.compareAndSet(false, true)) { + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "ZBS_PROVIDER_CALL_TIMEOUT: Provider[%s] Update timed out", + provider.getProviderType())); + } + }, + TimeUnit.MILLISECONDS, + timeout); + try { + provider.update(ref, update, new Completion(completion) { + @Override + public void success() { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.success(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.fail(errorCode); + } + } + }); + } catch (RuntimeException error) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.fail(operr( + ORG_ZSTACK_CORE_10000, + "ZBS_PROVIDER_CALL_FAILED: Provider[%s] Update failed: %s", + provider.getProviderType(), error.getMessage())); + } + } + } + + private long providerTimeoutMillis() { + return Math.max( + 1, + TimeUnit.SECONDS.toMillis( + ZbsResourceAssignmentGlobalConfig.PROVIDER_CALL_TIMEOUT + .value(Long.class))); + } + + private boolean isOwnerState(String state) { + return "READY".equals(state) + || "PENDING".equals(state) + || "UNSUPPORTED".equals(state) + || "ERROR".equals(state) + || "DISABLED".equals(state); + } + + private boolean empty(String value) { + return value == null || value.trim().isEmpty(); + } + + private String normalize(String value) { + return value == null ? "" : value.trim(); + } + + private boolean memoryMatches(Long expected, Long actual) { + return expected == null + || expected.equals(actual) + || expected == 0L && actual == null; + } + + private static class ProviderResolution { + private final ZbsCpuIsolationProvider provider; + private final String reasonCode; + + private ProviderResolution( + ZbsCpuIsolationProvider provider, + String reasonCode) { + this.provider = provider; + this.reasonCode = reasonCode; + } + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsResourceAssignmentGlobalConfig.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsResourceAssignmentGlobalConfig.java new file mode 100644 index 00000000000..7ef175e7724 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsResourceAssignmentGlobalConfig.java @@ -0,0 +1,17 @@ +package org.zstack.storage.zbs; + +import org.zstack.core.config.GlobalConfig; +import org.zstack.core.config.GlobalConfigDef; +import org.zstack.core.config.GlobalConfigDefinition; +import org.zstack.core.config.GlobalConfigValidation; + +@GlobalConfigDefinition +public class ZbsResourceAssignmentGlobalConfig { + public static final String CATEGORY = "physicalServer"; + + @GlobalConfigDef(type = Long.class, defaultValue = "60", + description = "ZBS CPU isolation Provider call timeout in seconds") + @GlobalConfigValidation(numberGreaterThan = 0) + public static GlobalConfig PROVIDER_CALL_TIMEOUT = + new GlobalConfig(CATEGORY, "zbsCpuIsolation.providerCallTimeout"); +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 74386981e87..12552c7706f 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -10,6 +10,7 @@ import org.zstack.compute.host.HostGlobalConfig; import org.zstack.compute.host.HostSystemTags; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.Platform; import org.zstack.core.ansible.AnsibleGlobalProperty; import org.zstack.core.asyncbatch.While; import org.zstack.core.cloudbus.CloudBus; @@ -92,6 +93,10 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar @Autowired @Deprecated private CloudBus bus; + @Autowired + private ZbsCpuIsolationCascadeExtension cpuIsolationCascade; + @Autowired + private ZbsPhysicalServerIdentityResolver physicalServerIdentities; private ExternalPrimaryStorageVO self; private AddonInfo addonInfo; @@ -632,6 +637,7 @@ public void connect(String cfg, String url, ReturnValueCompletion mdsInfos = MdsInfo.valueOf(current.getMdsUrls()); + preservePhysicalServerSerialNumbers(mdsInfos); newAddonInfo.setMdsInfos(mdsInfos); final List mdsList = CollectionUtils.transformToList(newAddonInfo.getMdsInfos(), ZbsPrimaryStorageMdsBase::new); @@ -781,6 +787,8 @@ public void done(ErrorCodeList errorCodeList) { done(new FlowDoneHandler(completion) { @Override public void handle(Map data) { + physicalServerIdentities.enrichPhysicalServerSerialNumbers( + self.getUuid(), newAddonInfo); addonInfo = newAddonInfo; completion.success(newAddonInfo); } @@ -1650,6 +1658,21 @@ public void syncConfig(String config) { this.config = StringUtils.isEmpty(config) ? new Config() : JSONObjectUtil.toObject(config, Config.class); } + @Override + public void beforePersistAddonInfo( + org.zstack.header.storage.addon.primary.AddonInfo addonInfo, + Completion completion) { + if (!(addonInfo instanceof AddonInfo)) { + completion.fail(operr( + ORG_ZSTACK_STORAGE_ZBS_10012, + "cannot update ZBS primary storage[uuid:%s]: invalid addonInfo type[%s]", + self.getUuid(), addonInfo == null ? null : addonInfo.getClass().getName())); + return; + } + cpuIsolationCascade.beforePersistAddonInfo( + self.getUuid(), (AddonInfo) addonInfo, completion); + } + private void syncMdsStatuses(AddonInfo newAddonInfo) { if (addonInfo == null || newAddonInfo == null) { return; @@ -1668,6 +1691,27 @@ private void syncMdsStatuses(AddonInfo newAddonInfo) { .update(); } + private void preservePhysicalServerSerialNumbers(List mdsInfos) { + if (addonInfo == null || addonInfo.getMdsInfos() == null) { + return; + } + Map serialNumbersByAddress = new HashMap<>(); + for (MdsInfo existing : addonInfo.getMdsInfos()) { + if (existing == null || existing.getAddr() == null) { + continue; + } + String serialNumber = Platform.normalizeMachineSerialNumber( + existing.getPhysicalServerSerialNumber()); + if (serialNumber != null) { + serialNumbersByAddress.put(existing.getAddr(), serialNumber); + } + } + for (MdsInfo mds : mdsInfos) { + mds.setPhysicalServerSerialNumber( + serialNumbersByAddress.get(mds.getAddr())); + } + } + @Deprecated private void reloadDbInfo() { self = dbf.reload(self); diff --git a/plugin/zbs/src/main/resources/physical-server-roles/zbs.yaml b/plugin/zbs/src/main/resources/physical-server-roles/zbs.yaml new file mode 100644 index 00000000000..b30977114b8 --- /dev/null +++ b/plugin/zbs/src/main/resources/physical-server-roles/zbs.yaml @@ -0,0 +1,4 @@ +roleType: ZBS +applicationMode: PROVIDER_MANAGED +services: + - name: zbs-node diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeResourceControlMsg.java b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeResourceControlMsg.java new file mode 100644 index 00000000000..243613e3ff8 --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeResourceControlMsg.java @@ -0,0 +1,25 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.physicalserver.ResourceControlCommand; + +public class ApplyManagementNodeResourceControlMsg extends NeedReplyMessage { + private String serverUuid; + private ResourceControlCommand command; + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public ResourceControlCommand getCommand() { + return command; + } + + public void setCommand(ResourceControlCommand command) { + this.command = command; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeResourceControlReply.java b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeResourceControlReply.java new file mode 100644 index 00000000000..83c17fd1e1d --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeResourceControlReply.java @@ -0,0 +1,16 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.ResourceControlResponse; + +public class ApplyManagementNodeResourceControlReply extends MessageReply { + private ResourceControlResponse response; + + public ResourceControlResponse getResponse() { + return response; + } + + public void setResponse(ResourceControlResponse response) { + this.response = response; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyMsg.java b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyMsg.java new file mode 100644 index 00000000000..6a18ec43880 --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyMsg.java @@ -0,0 +1,15 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.NeedReplyMessage; + +public class CollectManagementNodeCpuTopologyMsg extends NeedReplyMessage { + private String serverUuid; + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyReply.java b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyReply.java new file mode 100644 index 00000000000..78738ea96fa --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyReply.java @@ -0,0 +1,16 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; + +public class CollectManagementNodeCpuTopologyReply extends MessageReply { + private PhysicalServerCpuTopology topology; + + public PhysicalServerCpuTopology getTopology() { + return topology; + } + + public void setTopology(PhysicalServerCpuTopology topology) { + this.topology = topology; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeManagedServicesMsg.java b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeManagedServicesMsg.java new file mode 100644 index 00000000000..f4e420fe237 --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeManagedServicesMsg.java @@ -0,0 +1,24 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.NeedReplyMessage; + +public class CollectManagementNodeManagedServicesMsg extends NeedReplyMessage { + private String serverUuid; + private boolean includeAuxiliaryServices; + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public boolean isIncludeAuxiliaryServices() { + return includeAuxiliaryServices; + } + + public void setIncludeAuxiliaryServices(boolean includeAuxiliaryServices) { + this.includeAuxiliaryServices = includeAuxiliaryServices; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeManagedServicesReply.java b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeManagedServicesReply.java new file mode 100644 index 00000000000..420a81f07dd --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeManagedServicesReply.java @@ -0,0 +1,19 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; + +import java.util.ArrayList; +import java.util.List; + +public class CollectManagementNodeManagedServicesReply extends MessageReply { + private List services = new ArrayList<>(); + + public List getServices() { + return services; + } + + public void setServices(List services) { + this.services = services; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/LocalCpuTopologyCollector.java b/portal/src/main/java/org/zstack/portal/managementnode/LocalCpuTopologyCollector.java new file mode 100644 index 00000000000..afdaa99f3dc --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/LocalCpuTopologyCollector.java @@ -0,0 +1,146 @@ +package org.zstack.portal.managementnode; + +import org.zstack.core.CoreGlobalProperty; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerNumaNode; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +public class LocalCpuTopologyCollector { + private static final Path CPU_ROOT = Paths.get("/sys/devices/system/cpu"); + private static final Path NODE_ROOT = Paths.get("/sys/devices/system/node"); + private final Path cpuRoot; + private final Path nodeRoot; + private volatile PhysicalServerCpuTopology testTopology; + + public LocalCpuTopologyCollector() { + this(CPU_ROOT, NODE_ROOT); + } + + LocalCpuTopologyCollector(Path cpuRoot, Path nodeRoot) { + this.cpuRoot = cpuRoot; + this.nodeRoot = nodeRoot; + } + + public PhysicalServerCpuTopology collect() { + if (CoreGlobalProperty.UNIT_TEST_ON && testTopology != null) { + return testTopology; + } + try { + SortedSet online = readCpuSet(cpuRoot.resolve("online")); + List nodePaths = nodePaths(); + if (nodePaths.isEmpty()) { + return PhysicalServerCpuTopology.from( + Collections.singletonMap("0", node("0", online, online))); + } + + Map result = new LinkedHashMap<>(); + for (Path nodePath : nodePaths) { + String nodeId = nodePath.getFileName().toString().substring("node".length()); + SortedSet cpus = readCpuSet(nodePath.resolve("cpulist")); + SortedSet nodeOnline = new TreeSet<>(cpus); + nodeOnline.retainAll(online); + if (!nodeOnline.isEmpty()) { + result.put(nodeId, node(nodeId, cpus, nodeOnline)); + } + } + if (result.isEmpty()) { + throw new IllegalArgumentException("CPU_TOPOLOGY_UNAVAILABLE: no online NUMA CPU found"); + } + return PhysicalServerCpuTopology.from(result); + } catch (IOException exception) { + throw new IllegalArgumentException("CPU_TOPOLOGY_UNAVAILABLE: " + exception.getMessage(), exception); + } + } + + public void setTestTopology(PhysicalServerCpuTopology testTopology) { + if (!CoreGlobalProperty.UNIT_TEST_ON) { + throw new IllegalStateException("test topology is only available in unit-test mode"); + } + this.testTopology = testTopology; + } + + public void clearTestTopology() { + testTopology = null; + } + + private List nodePaths() throws IOException { + if (!Files.isDirectory(nodeRoot)) { + return Collections.emptyList(); + } + List paths = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(nodeRoot, "node[0-9]*")) { + for (Path path : stream) { + if (path.getFileName().toString().matches("node[0-9]+")) { + paths.add(path); + } + } + } + paths.sort(Comparator.comparingInt(path -> Integer.parseInt( + path.getFileName().toString().substring("node".length())))); + return paths; + } + + private PhysicalServerNumaNode node( + String nodeId, SortedSet cpus, SortedSet online) throws IOException { + PhysicalServerNumaNode node = new PhysicalServerNumaNode(); + node.setNodeId(nodeId); + node.setOnlineCpus(strings(online)); + node.setCoreGroups(coreGroups(online)); + return node; + } + + private List> coreGroups(SortedSet online) throws IOException { + Set> groups = new LinkedHashSet<>(); + for (Integer cpu : online) { + Path siblings = cpuRoot.resolve(String.format( + "cpu%s/topology/thread_siblings_list", cpu)); + SortedSet group = readCpuSet(siblings); + group.retainAll(online); + if (group.isEmpty()) { + throw new IllegalArgumentException(String.format( + "CPU_TOPOLOGY_INVALID: CPU[%s] has no online core sibling", cpu)); + } + groups.add(group); + } + List> sorted = new ArrayList<>(groups); + sorted.sort(Comparator.comparingInt(SortedSet::first)); + List> result = new ArrayList<>(); + for (SortedSet group : sorted) { + result.add(strings(group)); + } + return result; + } + + private SortedSet readCpuSet(Path path) throws IOException { + String value = new String(Files.readAllBytes(path), StandardCharsets.US_ASCII).trim(); + if (value.isEmpty()) { + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: empty CPU list at " + path); + } + return PhysicalServerCpuSet.parse(value); + } + + private List strings(Set cpus) { + List result = new ArrayList<>(); + for (Integer cpu : cpus) { + result.add(String.valueOf(cpu)); + } + return result; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/LocalResourceControlExecutor.java b/portal/src/main/java/org/zstack/portal/managementnode/LocalResourceControlExecutor.java new file mode 100644 index 00000000000..dc4ad2b323c --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/LocalResourceControlExecutor.java @@ -0,0 +1,1733 @@ +package org.zstack.portal.managementnode; + +import org.zstack.core.CoreGlobalProperty; +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.ResourceConsumerHandle; +import org.zstack.header.physicalserver.ResourceControlCommand; +import org.zstack.header.physicalserver.ResourceControlResponse; +import org.zstack.header.physicalserver.ResourceControlResult; +import org.zstack.utils.data.SizeUnit; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class LocalResourceControlExecutor { + private static final String SYSTEMD_DROP_IN = + "50-zstack-resource-assignment.conf"; + private static final long COMMAND_TIMEOUT_SECONDS = 30; + private static final int PROCESS_MOVE_ATTEMPTS = 3; + + private final AtomicInteger testCalls = new AtomicInteger(); + private volatile boolean testMode; + private volatile ResourceControlCommand lastTestCommand; + private final AtomicReference> + lastTestRestartHandles = new AtomicReference<>(); + private final ExecutionEnvironment environment; + + public LocalResourceControlExecutor() { + this(ExecutionEnvironment.system()); + } + + LocalResourceControlExecutor(ExecutionEnvironment environment) { + this.environment = environment; + } + + public ResourceControlResponse apply(ResourceControlCommand command) { + if (CoreGlobalProperty.UNIT_TEST_ON && testMode) { + return fakeApply(command); + } + + boolean enabled = !"RELEASE".equals(command.getOperation()); + Backend backend; + try { + backend = backend(); + } catch (ResourceControlException exception) { + if ("RESOURCE_CONTROL_UNAVAILABLE".equals(reasonCode(exception))) { + return unavailable(command); + } + throw exception; + } + String desired = enabled + ? PhysicalServerCpuSet.normalize(command.getCpuSet()) : ""; + validateMemoryLimit(command.getMemory()); + Long desiredMemory = command.getMemory() == null + ? null : enabled ? command.getMemory() : 0L; + MemoryBackend memoryBackend = null; + if (command.getMemory() != null) { + try { + memoryBackend = memoryBackend(); + } catch (ResourceControlException exception) { + if (!"MEMORY_CONTROLLER_UNAVAILABLE".equals( + reasonCode(exception))) { + throw exception; + } + } + } + if (command.getSliceName() != null + && command.getHandles().stream().anyMatch(handle -> + ResourceConsumerHandle.SYSTEMD_UNIT.equals( + handle.getHandleType()))) { + return applySystemdSlice( + command, backend, memoryBackend, + desired, desiredMemory, enabled); + } + List results = new ArrayList<>(); + + for (ResourceConsumerHandle handle : command.getHandles()) { + Path target; + try { + target = resolve( + backend, command.getRoleType(), handle, enabled); + } catch (ResourceControlException exception) { + results.add(result("ERROR", null, null)); + continue; + } + if (target == null) { + results.add(result("SKIPPED", null, null)); + continue; + } + try { + String actualCpuSet = applyToGroup( + backend, target, desired, enabled); + Long actualMemory = null; + if (command.getMemory() != null) { + actualMemory = applyMemoryLimit( + backend, memoryBackend, + target, desiredMemory).actualLimit; + } + results.add(result( + enabled ? "READY" : "DISABLED", + actualCpuSet, + actualMemory)); + } catch (ResourceControlException exception) { + results.add(result("ERROR", null, null)); + } + } + return summarize(results); + } + + private ResourceControlResponse applySystemdSlice( + ResourceControlCommand command, + Backend backend, + MemoryBackend memoryBackend, + String desired, + Long desiredMemory, + boolean enabled) { + if (enabled && desiredMemory != null && desiredMemory > 0) { + if (memoryBackend != null) { + validateActiveSliceMemory( + memoryBackend, + command.getSliceName(), desiredMemory); + } + } + boolean changed = configureSystemdSlice( + backend, + memoryBackend, + command.getSliceName(), + desired, + desiredMemory, + enabled, + command.getMemory() != null); + for (ResourceConsumerHandle handle : command.getHandles()) { + if (ResourceConsumerHandle.SYSTEMD_UNIT.equals( + handle.getHandleType())) { + changed = configureSystemdService( + handle, command.getSliceName(), enabled) || changed; + } + } + boolean legacyCpuFallback = false; + Map legacyCpuResults = new HashMap<>(); + Path sliceTarget; + try { + sliceTarget = activeSliceTarget( + backend, command.getSliceName(), enabled); + } catch (ResourceControlException exception) { + if (!"SYSTEMD_CONTROL_GROUP_NOT_FOUND".equals( + reasonCode(exception))) { + throw exception; + } + legacyCpuFallback = true; + sliceTarget = null; + for (int index = 0; index < command.getHandles().size(); index++) { + ResourceConsumerHandle handle = command.getHandles().get(index); + if (!ResourceConsumerHandle.SYSTEMD_UNIT.equals( + handle.getHandleType())) { + continue; + } + legacyCpuResults.put(index, applyNonSystemdHandle( + command, backend, null, handle, desired, + null, enabled, false)); + } + } + String actualCpuSet = ""; + Long actualMemory = null; + Path memorySliceTarget = null; + boolean memoryError = command.getMemory() != null + && memoryBackend == null; + if (sliceTarget != null) { + actualCpuSet = applyToGroup( + backend, sliceTarget, desired, enabled); + } + if (command.getMemory() != null && !memoryError) { + try { + memorySliceTarget = activeControllerSliceTarget( + memoryBackend.root, command.getSliceName()); + if (memorySliceTarget != null) { + actualMemory = applyMemoryTarget( + memoryBackend, memorySliceTarget, + desiredMemory, false, null).actualLimit; + } + } catch (ResourceControlException exception) { + memoryError = true; + } + } + if (changed) { + run(null, "sudo", "-n", "systemctl", "daemon-reload"); + } + + List results = new ArrayList<>(); + for (int index = 0; index < command.getHandles().size(); index++) { + ResourceConsumerHandle handle = command.getHandles().get(index); + if (!ResourceConsumerHandle.SYSTEMD_UNIT.equals( + handle.getHandleType())) { + results.add(applyNonSystemdHandle( + command, backend, memoryBackend, handle, desired, + desiredMemory, enabled, + command.getMemory() != null)); + continue; + } + Map properties = systemdProperties(handle.getValue()); + if ("not-found".equals(properties.get("LoadState"))) { + results.add(result( + handle.isOptional() ? "SKIPPED" : "ERROR", + null, null)); + continue; + } + if (!"active".equals(properties.get("ActiveState"))) { + results.add(result( + handle.isOptional() ? "SKIPPED" : "ERROR", + null, null)); + continue; + } + if (!enabled) { + results.add(memoryError + ? result("ERROR", null, null) + : result("DISABLED", "", + command.getMemory() == null ? null : 0L)); + continue; + } + String serviceCpuSet; + if (legacyCpuFallback) { + ResourceControlResult cpuResult = legacyCpuResults.get(index); + if (cpuResult == null || "ERROR".equals(cpuResult.getState()) + || "SKIPPED".equals(cpuResult.getState())) { + results.add(cpuResult == null + ? result("ERROR", null, null) : cpuResult); + continue; + } + serviceCpuSet = cpuResult.getCpuSet(); + } else { + Path current = systemdTarget( + backend, properties.get("ControlGroup")); + if (sliceTarget == null || !current.startsWith(sliceTarget)) { + results.add(result("PENDING_RESTART", null, null)); + continue; + } + serviceCpuSet = actualCpuSet; + } + if (memoryError) { + results.add(result("ERROR", null, null)); + continue; + } + if (command.getMemory() != null && memorySliceTarget != null + && !controlGroupInTarget( + memoryBackend.root, + properties.get("ControlGroup"), + memorySliceTarget)) { + results.add(result("PENDING_RESTART", null, null)); + continue; + } + results.add(result("READY", serviceCpuSet, actualMemory)); + } + ResourceControlResponse response = summarize(results); + if (sliceTarget != null) { + response.setCpuSet(actualCpuSet); + } + if (command.getMemory() != null && !memoryError) { + response.setMemory(actualMemory); + } + return response; + } + + private void validateActiveSliceMemory( + MemoryBackend memoryBackend, + String sliceName, + long desiredMemory) { + Map properties = systemdProperties(sliceName); + if (!"active".equals(properties.get("ActiveState"))) { + return; + } + Path memoryTarget = systemdTarget( + memoryBackend.root, properties.get("ControlGroup")); + validateMemoryLimitAgainstUsage( + memoryTarget.resolve(memoryBackend.v2 + ? "memory.current" : "memory.usage_in_bytes"), + desiredMemory); + } + + private ResourceControlResult applyNonSystemdHandle( + ResourceControlCommand command, + Backend backend, + MemoryBackend memoryBackend, + ResourceConsumerHandle handle, + String desired, + Long desiredMemory, + boolean enabled, + boolean manageMemory) { + try { + Path target = resolve( + backend, command.getRoleType(), handle, enabled); + if (target == null) { + return result("SKIPPED", null, null); + } + String actualCpuSet = applyToGroup( + backend, target, desired, enabled); + Long actualMemory = null; + if (manageMemory) { + actualMemory = applyMemoryLimit( + backend, memoryBackend, + target, desiredMemory).actualLimit; + } + return result( + enabled ? "READY" : "DISABLED", + actualCpuSet, actualMemory); + } catch (ResourceControlException exception) { + return result("ERROR", null, null); + } + } + + private boolean configureSystemdSlice( + Backend backend, + MemoryBackend memoryBackend, + String sliceName, + String cpuSet, + Long memory, + boolean enabled, + boolean manageMemory) { + Path path = dropInPath(sliceName); + if (!enabled) { + return removeDropIn(path); + } + List lines = new ArrayList<>(); + lines.add("[Slice]"); + if (backend.v2) { + lines.add("AllowedCPUs=" + cpuSet); + } + if (manageMemory && memoryBackend != null) { + lines.add(String.format( + "%s=%s", + memoryBackend.v2 ? "MemoryMax" : "MemoryLimit", + memory == 0 ? "infinity" : memory)); + } else if (manageMemory && Files.isRegularFile(path)) { + for (String line : read(path).split("\\R")) { + if (line.startsWith("MemoryMax=") + || line.startsWith("MemoryLimit=")) { + lines.add(line); + } + } + } + return writeDropIn(path, String.join("\n", lines) + "\n"); + } + + private boolean configureSystemdService( + ResourceConsumerHandle handle, + String sliceName, + boolean enabled) { + Path path = dropInPath(handle.getValue()); + if (!enabled) { + return removeDropIn(path); + } + return writeDropIn( + path, "[Service]\nSlice=" + sliceName + "\n"); + } + + private Path dropInPath(String unit) { + return environment.systemdUnitRoot.resolve(unit + ".d") + .resolve(SYSTEMD_DROP_IN); + } + + private boolean writeDropIn(Path path, String content) { + if (Files.isRegularFile(path) + && content.equals(read(path))) { + return false; + } + Path temporary = null; + try { + temporary = Files.createTempFile( + "zstack-resource-assignment-", ".conf"); + Files.write(temporary, content.getBytes(StandardCharsets.US_ASCII)); + run(null, "sudo", "-n", "mkdir", "-p", + path.getParent().toString()); + run(null, "sudo", "-n", "install", "-m", "0644", + temporary.toString(), path.toString()); + return true; + } catch (IOException exception) { + throw new ResourceControlException( + "SYSTEMD_DROP_IN_WRITE_FAILED:" + exception.getMessage()); + } finally { + if (temporary != null) { + try { + Files.deleteIfExists(temporary); + } catch (IOException ignored) { + } + } + } + } + + private boolean removeDropIn(Path path) { + if (!Files.exists(path)) { + return false; + } + run(null, "sudo", "-n", "rm", "-f", path.toString()); + return true; + } + + private Path activeSliceTarget( + Backend backend, String sliceName, boolean enabled) { + Map properties = systemdProperties(sliceName); + if (enabled && !"active".equals(properties.get("ActiveState"))) { + run(null, "sudo", "-n", "systemctl", "start", sliceName); + properties = systemdProperties(sliceName); + } + if (!"active".equals(properties.get("ActiveState"))) { + return null; + } + return systemdTarget(backend, properties.get("ControlGroup")); + } + + private Path activeControllerSliceTarget( + Path root, String sliceName) { + Map properties = systemdProperties(sliceName); + if (!"active".equals(properties.get("ActiveState"))) { + return null; + } + return systemdTarget(root, properties.get("ControlGroup")); + } + + private Path systemdTarget(Backend backend, String controlGroup) { + return systemdTarget(backend.root, controlGroup); + } + + private Path systemdTarget(Path root, String controlGroup) { + if (controlGroup == null || controlGroup.isEmpty()) { + throw new ResourceControlException("SYSTEMD_CONTROL_GROUP_MISSING"); + } + Path target = underRoot( + root, + root.resolve(stripRoot(controlGroup)).normalize()); + if (target.equals(root) || !Files.isDirectory(target)) { + throw new ResourceControlException( + "SYSTEMD_CONTROL_GROUP_NOT_FOUND"); + } + return target; + } + + private boolean controlGroupInTarget( + Path root, String controlGroup, Path target) { + try { + return systemdTarget(root, controlGroup).startsWith(target); + } catch (ResourceControlException exception) { + return false; + } + } + + private ResourceControlResponse unavailable(ResourceControlCommand command) { + List results = new ArrayList<>(); + for (ResourceConsumerHandle handle : command.getHandles()) { + results.add(result("ERROR", null, null)); + } + return summarize(results); + } + + public List inspect( + String roleType, List handles) { + if (CoreGlobalProperty.UNIT_TEST_ON && testMode) { + return fakeInspect(handles); + } + Backend backend; + try { + backend = backend(); + } catch (ResourceControlException exception) { + List unavailable = new ArrayList<>(); + for (ResourceConsumerHandle handle : handles) { + unavailable.add(serviceUsage(handle, "UNAVAILABLE")); + } + return unavailable; + } + List result = new ArrayList<>(); + for (ResourceConsumerHandle handle : handles) { + ManagedServiceResourceUsage usage = serviceUsage( + handle, "UNAVAILABLE"); + try { + ServiceTarget target = inspectTarget( + backend, roleType, handle); + usage.setState(target.state); + if (target.path != null) { + fillUsage(usage, backend, target.path, handle); + } + } catch (ResourceControlException ignored) { + } + result.add(usage); + } + return result; + } + + public void restart(List handles) { + if (handles == null || handles.isEmpty()) { + throw new ResourceControlException("SERVICE_HANDLE_SET_INVALID"); + } + for (ResourceConsumerHandle handle : handles) { + if (!ResourceConsumerHandle.SYSTEMD_UNIT.equals( + handle.getHandleType()) || !handle.isRestartable()) { + throw new ResourceControlException( + "SERVICE_RESTART_NOT_ALLOWED"); + } + } + if (CoreGlobalProperty.UNIT_TEST_ON && testMode) { + lastTestRestartHandles.set(new ArrayList<>(handles)); + return; + } + List units = new ArrayList<>(); + for (ResourceConsumerHandle handle : handles) { + Map properties = systemdProperties(handle.getValue()); + if ("not-found".equals(properties.get("LoadState"))) { + throw new ResourceControlException("SYSTEMD_UNIT_NOT_FOUND"); + } + if (!"active".equals(properties.get("ActiveState"))) { + throw new ResourceControlException("SYSTEMD_UNIT_NOT_ACTIVE"); + } + units.add(handle.getValue()); + } + List stop = new ArrayList<>(Arrays.asList( + "sudo", "-n", "systemctl", "stop")); + stop.addAll(units); + run(null, stop.toArray(new String[0])); + List start = new ArrayList<>(Arrays.asList( + "sudo", "-n", "systemctl", "start")); + start.addAll(units); + run(null, start.toArray(new String[0])); + for (ResourceConsumerHandle handle : handles) { + if (!"active".equals( + systemdProperties(handle.getValue()).get("ActiveState"))) { + throw new ResourceControlException( + "SYSTEMD_UNIT_RESTART_FAILED"); + } + } + } + + private List fakeInspect( + List handles) { + List result = new ArrayList<>(); + for (ResourceConsumerHandle handle : handles) { + ManagedServiceResourceUsage usage = serviceUsage(handle, "RUNNING"); + if (lastTestCommand != null) { + usage.setCpuSet(lastTestCommand.getCpuSet()); + usage.setMemoryLimit(lastTestCommand.getMemory()); + } + result.add(usage); + } + return result; + } + + private ManagedServiceResourceUsage serviceUsage( + ResourceConsumerHandle handle, String state) { + ManagedServiceResourceUsage usage = new ManagedServiceResourceUsage(); + usage.setServiceName(handle.getServiceName()); + usage.setRestartable(handle.isRestartable()); + usage.setState(state); + return usage; + } + + private ServiceTarget inspectTarget( + Backend backend, + String roleType, + ResourceConsumerHandle handle) { + if (ResourceConsumerHandle.SYSTEMD_UNIT.equals(handle.getHandleType())) { + Map properties = systemdProperties(handle.getValue()); + if ("not-found".equals(properties.get("LoadState"))) { + return new ServiceTarget("NOT_FOUND", null); + } + if (!"active".equals(properties.get("ActiveState"))) { + return new ServiceTarget("INACTIVE", null); + } + String controlGroup = properties.get("ControlGroup"); + if (controlGroup != null && !controlGroup.isEmpty()) { + Path current = underRoot( + backend.root, + backend.root.resolve( + stripRoot(controlGroup)).normalize()); + if (Files.isDirectory(current)) { + return new ServiceTarget("RUNNING", current); + } + } + Path managed = backend.root.resolve(String.format( + "zstack-role-%s-unit-%s", + safeRole(roleType), safeRole(handle.getValue()))); + if (groupHasProcesses(managed)) { + return new ServiceTarget("RUNNING", managed); + } + String mainPid = properties.get("MainPID"); + if (mainPid != null && mainPid.matches("[1-9][0-9]*")) { + return new ServiceTarget( + "RUNNING", processGroup(backend, mainPid)); + } + throw new ResourceControlException( + "SYSTEMD_CONTROL_GROUP_NOT_FOUND"); + } + throw new ResourceControlException("HANDLE_TYPE_UNSUPPORTED"); + } + + private boolean groupHasProcesses(Path target) { + return Files.isDirectory(target) + && Files.isRegularFile(target.resolve("cgroup.procs")) + && !processIds(target.resolve("cgroup.procs")).isEmpty(); + } + + private void fillUsage( + ManagedServiceResourceUsage usage, + Backend backend, + Path target, + ResourceConsumerHandle handle) { + usage.setCpuSet(effectiveCpuSet(backend, target)); + Path relative = backend.root.relativize(target); + String controlGroup = null; + if (ResourceConsumerHandle.SYSTEMD_UNIT.equals( + handle.getHandleType())) { + controlGroup = systemdProperties( + handle.getValue()).get("ControlGroup"); + } + usage.setCpuTime(cpuTime(relative, controlGroup)); + MemoryBackend memoryBackend; + try { + memoryBackend = memoryBackend(); + } catch (ResourceControlException exception) { + if (!"MEMORY_CONTROLLER_UNAVAILABLE".equals( + reasonCode(exception))) { + throw exception; + } + return; + } + Path memoryTarget = controllerTarget(memoryBackend.root, relative); + if (controlGroup != null && !controlGroup.isEmpty()) { + Path current = controllerTarget( + memoryBackend.root, + Paths.get(stripRoot(controlGroup))); + if (Files.isDirectory(current)) { + memoryTarget = current; + } + } + if (!Files.isDirectory(memoryTarget)) { + return; + } + Path current = memoryTarget.resolve(memoryBackend.v2 + ? "memory.current" : "memory.usage_in_bytes"); + usage.setMemory(Files.isRegularFile(current) + ? parseMemoryLimit(read(current).trim()) : null); + usage.setMemoryLimit(memoryBackend.v2 + ? effectiveV2MemoryLimit(memoryBackend.root, memoryTarget) + : effectiveV1MemoryLimit( + memoryBackend.root, memoryTarget)); + } + + private Long cpuTime(Path relative, String controlGroup) { + if (!relative.toString().isEmpty()) { + for (Path root : v2Roots()) { + Long value = v2CpuTime(controllerTarget(root, relative)); + if (value != null) { + return value; + } + } + Long value = v1CpuTime(relative); + if (value != null) { + return value; + } + } + if (controlGroup == null || controlGroup.isEmpty()) { + return null; + } + return v1CpuTime(Paths.get(stripRoot(controlGroup))); + } + + private String effectiveCpuSet(Backend backend, Path target) { + Path current = target; + while (true) { + Path effective = current.resolve("cpuset.cpus.effective"); + Path configured = current.resolve("cpuset.cpus"); + for (Path candidate : Arrays.asList(effective, configured)) { + if (!Files.isRegularFile(candidate)) { + continue; + } + String value = normalizeOptional(read(candidate)); + if (!value.isEmpty()) { + return value; + } + } + if (current.equals(backend.root)) { + throw new ResourceControlException( + "RESOURCE_CONTROL_CPUSET_EMPTY"); + } + current = underRoot(backend.root, current.getParent()); + } + } + + private Path processGroup(Backend backend, String pid) { + for (String line : read(Paths.get("/proc", pid, "cgroup")).split("\\R")) { + String[] fields = line.split(":", 3); + if (fields.length != 3) { + continue; + } + boolean matches = backend.v2 && "0".equals(fields[0]); + if (!backend.v2) { + matches = Arrays.asList(fields[1].split(",")) + .contains("cpuset"); + } + if (!matches) { + continue; + } + Path target = underRoot( + backend.root, + backend.root.resolve(stripRoot(fields[2])).normalize()); + if (Files.isDirectory(target)) { + return target; + } + } + throw new ResourceControlException( + "PROCESS_CONTROL_GROUP_NOT_FOUND"); + } + + private Long v2CpuTime(Path target) { + Path stat = target.resolve("cpu.stat"); + if (!Files.isRegularFile(stat)) { + return null; + } + for (String line : read(stat).split("\\R")) { + String[] fields = line.trim().split("\\s+"); + if (fields.length == 2 && "usage_usec".equals(fields[0])) { + return Math.multiplyExact( + parseMemoryLimit(fields[1]), 1000L); + } + } + return null; + } + + private Long v1CpuTime(Path relative) { + for (Path root : environment.v1CpuacctRoots) { + Path usage = root.resolve(relative).resolve("cpuacct.usage"); + if (Files.isRegularFile(usage)) { + return parseMemoryLimit(read(usage).trim()); + } + } + return null; + } + + private Long effectiveV2MemoryLimit(Path root, Path target) { + Long effective = null; + Path current = target; + while (true) { + Path limit = current.resolve("memory.max"); + if (Files.isRegularFile(limit)) { + String value = read(limit).trim(); + if (!"max".equals(value)) { + long parsed = parseMemoryLimit(value); + effective = effective == null + ? parsed : Math.min(effective, parsed); + } + } + if (current.equals(root)) { + break; + } + current = underRoot(root, current.getParent()); + } + return effective == null ? 0L : effective; + } + + private Long effectiveV1MemoryLimit(Path root, Path target) { + Path rootLimitPath = root.resolve("memory.limit_in_bytes"); + if (!Files.isRegularFile(rootLimitPath)) { + return null; + } + long rootLimit = parseMemoryLimit(read(rootLimitPath).trim()); + underRoot(root, target); + Long effective = null; + Path current = target; + while (true) { + Path limit = current.resolve("memory.limit_in_bytes"); + if (Files.isRegularFile(limit)) { + long parsed = parseMemoryLimit(read(limit).trim()); + effective = effective == null + ? parsed : Math.min(effective, parsed); + } + if (current.equals(root)) { + break; + } + current = underRoot(root, current.getParent()); + } + if (effective == null) { + return null; + } + return effective >= rootLimit ? 0L : effective; + } + + public void setTestMode(boolean testMode) { + if (!CoreGlobalProperty.UNIT_TEST_ON) { + throw new IllegalStateException("test executor is only available in unit-test mode"); + } + this.testMode = testMode; + this.lastTestCommand = null; + this.lastTestRestartHandles.set(null); + this.testCalls.set(0); + } + + public ResourceControlCommand getLastTestCommand() { + return lastTestCommand; + } + + public int getTestCalls() { + return testCalls.get(); + } + + public List getLastTestRestartHandles() { + List handles = lastTestRestartHandles.get(); + return handles == null ? null : new ArrayList<>(handles); + } + + private ResourceControlResponse fakeApply(ResourceControlCommand command) { + lastTestCommand = command; + testCalls.incrementAndGet(); + boolean enabled = !"RELEASE".equals(command.getOperation()); + String actualCpuSet = enabled + ? PhysicalServerCpuSet.normalize(command.getCpuSet()) : ""; + Long actualMemory = null; + if (command.getMemory() != null) { + validateMemoryLimit(command.getMemory()); + actualMemory = enabled ? command.getMemory() : 0L; + } + List results = new ArrayList<>(); + for (ResourceConsumerHandle handle : command.getHandles()) { + results.add(result( + enabled ? "READY" : "DISABLED", + actualCpuSet, + actualMemory)); + } + return summarize(results); + } + + private ResourceControlResponse summarize(List results) { + int expected = 0; + int covered = 0; + Set actualSets = new LinkedHashSet<>(); + Set actualMemory = new LinkedHashSet<>(); + + for (ResourceControlResult result : results) { + if ("SKIPPED".equals(result.getState())) { + continue; + } + expected++; + if ("READY".equals(result.getState()) + || "DISABLED".equals(result.getState())) { + covered++; + if (result.getCpuSet() != null) { + actualSets.add(result.getCpuSet()); + } + if (result.getMemory() != null) { + actualMemory.add(result.getMemory()); + } + } + } + + String cpuSet = actualSets.size() == 1 + ? actualSets.iterator().next() : ""; + Long memory = actualMemory.size() == 1 + ? actualMemory.iterator().next() : null; + + ResourceControlResponse response = new ResourceControlResponse(); + response.setCpuSet(cpuSet); + response.setMemory(memory); + response.setCoveredServiceCount(covered); + response.setExpectedServiceCount(expected); + response.setResults(results); + return response; + } + + private ResourceControlResult result( + String state, + String cpuSet, + Long memory) { + ResourceControlResult result = new ResourceControlResult(); + result.setState(state); + result.setCpuSet(cpuSet); + result.setMemory(memory); + return result; + } + + private Backend backend() { + for (Path root : v2Roots()) { + Path controllers = root.resolve("cgroup.controllers"); + String values = read(controllers); + if (values.matches("(?s).*\\bcpuset\\b.*") + || Files.isRegularFile( + root.resolve("cpuset.cpus.effective"))) { + return new Backend("CGROUP_V2_CPUSET", root, true); + } + } + if (Files.isRegularFile( + environment.v1Root.resolve("cpuset.cpus"))) { + return new Backend( + "CGROUP_V1_CPUSET", environment.v1Root, false); + } + throw new ResourceControlException("RESOURCE_CONTROL_UNAVAILABLE"); + } + + private MemoryBackend memoryBackend() { + for (Path root : v2Roots()) { + String controllers = read(root.resolve("cgroup.controllers")); + if (controllers.matches("(?s).*\\bmemory\\b.*") + || Files.isRegularFile(root.resolve("memory.max"))) { + return new MemoryBackend( + "CGROUP_V2_MEMORY", root, true); + } + } + if (Files.isRegularFile( + environment.v1MemoryRoot.resolve("memory.limit_in_bytes"))) { + return new MemoryBackend( + "CGROUP_V1_MEMORY", environment.v1MemoryRoot, false); + } + throw new ResourceControlException("MEMORY_CONTROLLER_UNAVAILABLE"); + } + + private List v2Roots() { + Set roots = new LinkedHashSet<>(); + if (Files.isRegularFile( + environment.v2Root.resolve("cgroup.controllers"))) { + roots.add(environment.v2Root); + } + if (!Files.isRegularFile(environment.procMounts)) { + return new ArrayList<>(roots); + } + for (String line : read(environment.procMounts).split("\\R")) { + String[] fields = line.trim().split("\\s+"); + if (fields.length < 4 || !"cgroup2".equals(fields[2]) + || !Arrays.asList(fields[3].split(",")).contains("rw")) { + continue; + } + Path root = Paths.get(decodeMountPath(fields[1])); + if (Files.isRegularFile(root.resolve("cgroup.controllers"))) { + roots.add(root); + } + } + return new ArrayList<>(roots); + } + + private Path controllerTarget(Path root, Path relative) { + return underRoot(root, root.resolve(relative).normalize()); + } + + private Path resolve( + Backend backend, + String roleType, + ResourceConsumerHandle handle, + boolean enabled) { + if (ResourceConsumerHandle.SYSTEMD_UNIT.equals(handle.getHandleType())) { + return resolveSystemd(backend, roleType, handle, enabled); + } + if (ResourceConsumerHandle.OWNER_PID_FILE.equals(handle.getHandleType())) { + return resolvePidFile(backend, roleType, handle, enabled); + } + throw new ResourceControlException("HANDLE_TYPE_UNSUPPORTED"); + } + + private Path resolveSystemd( + Backend backend, + String roleType, + ResourceConsumerHandle handle, + boolean enabled) { + Path managedTarget = backend.root.resolve(String.format( + "zstack-role-%s-unit-%s", + safeRole(roleType), safeRole(handle.getValue()))); + if (!enabled && Files.isDirectory(managedTarget)) { + return managedTarget; + } + Map properties = systemdProperties(handle.getValue()); + if ("not-found".equals(properties.get("LoadState"))) { + if (handle.isOptional()) { + return null; + } + throw new ResourceControlException("SYSTEMD_UNIT_NOT_FOUND"); + } + if (!"active".equals(properties.get("ActiveState"))) { + if (!enabled && handle.isOptional()) { + return null; + } + throw new ResourceControlException("SYSTEMD_UNIT_NOT_ACTIVE"); + } + String controlGroup = properties.get("ControlGroup"); + if (controlGroup == null || controlGroup.isEmpty()) { + throw new ResourceControlException("SYSTEMD_CONTROL_GROUP_MISSING"); + } + Path target = underRoot(backend.root, backend.root.resolve(stripRoot(controlGroup)).normalize()); + if (target.equals(backend.root)) { + throw new ResourceControlException("SYSTEMD_CONTROL_GROUP_INVALID"); + } + if (!Files.isDirectory(target)) { + return resolveSystemdV1Fallback( + backend, handle, controlGroup, enabled, managedTarget); + } + return target; + } + + private Path resolveSystemdV1Fallback( + Backend backend, + ResourceConsumerHandle handle, + String controlGroup, + boolean enabled, + Path target) { + if (!enabled) { + return null; + } + Path source = underRoot( + environment.v1SystemdRoot, + environment.v1SystemdRoot.resolve( + stripRoot(controlGroup)).normalize()); + Path sourceProcesses = source.resolve("cgroup.procs"); + if (!Files.isRegularFile(sourceProcesses)) { + if (handle.isOptional()) { + return null; + } + throw new ResourceControlException("SYSTEMD_CONTROL_GROUP_NOT_FOUND"); + } + if (processIds(sourceProcesses).isEmpty()) { + if (handle.isOptional()) { + return null; + } + throw new ResourceControlException("SYSTEMD_CONTROL_GROUP_EMPTY"); + } + mkdir(target); + enableV2Path(backend, target); + initializeMems(backend, target); + initializeCpus(backend, target); + Path targetProcesses = target.resolve("cgroup.procs"); + moveProcesses( + sourceProcesses, + targetProcesses, + "SYSTEMD_CONTROL_GROUP_NOT_FOUND", + "SYSTEMD_PROCESS_MOVE_FAILED"); + return target; + } + + private Path resolvePidFile( + Backend backend, + String roleType, + ResourceConsumerHandle handle, + boolean enabled) { + Path target = backend.root.resolve(String.format( + "zstack-role-%s-owner-%s", + safeRole(roleType), + safeRole(handle.getConsumerKey() == null + ? handle.getValue() : handle.getConsumerKey()))); + if (!enabled) { + return Files.isDirectory(target) ? target : null; + } + Path pidFile = Paths.get(handle.getValue()); + if (!Files.exists(pidFile, LinkOption.NOFOLLOW_LINKS)) { + if (handle.isOptional()) { + return null; + } + throw new ResourceControlException("PID_FILE_NOT_FOUND"); + } + try { + PosixFileAttributes attributes = Files.readAttributes( + pidFile, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (!attributes.isRegularFile() || Files.isSymbolicLink(pidFile)) { + throw new ResourceControlException("PID_FILE_NOT_REGULAR"); + } + String content = read(pidFile).trim(); + if (!content.matches("[1-9][0-9]*")) { + throw new ResourceControlException("PID_FILE_FORMAT_INVALID"); + } + long parsed = Long.parseLong(content); + if (parsed > Integer.MAX_VALUE) { + throw new ResourceControlException("PID_FILE_FORMAT_INVALID"); + } + int pid = (int) parsed; + Path proc = Paths.get("/proc", String.valueOf(pid)); + if (!Files.isDirectory(proc)) { + throw new ResourceControlException("PID_NOT_RUNNING"); + } + if (attributes.lastModifiedTime().toMillis() + 1000 < processStartTimeMillis(pid)) { + throw new ResourceControlException("PID_FILE_STALE"); + } + PosixFileAttributes procAttributes = Files.readAttributes( + proc, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + String expected = handle.getExpectedCommandToken(); + Set permissions = attributes.permissions(); + boolean writableByOthers = permissions.contains(PosixFilePermission.GROUP_WRITE) + || permissions.contains(PosixFilePermission.OTHERS_WRITE); + boolean trustedRootPidFile = "root".equals(attributes.owner().getName()) + && "root".equals(procAttributes.owner().getName()) + && expected != null && !expected.isEmpty(); + if (writableByOthers && !trustedRootPidFile) { + throw new ResourceControlException("PID_FILE_PERMISSION_INVALID"); + } + if (!"root".equals(attributes.owner().getName()) + && !attributes.owner().equals(procAttributes.owner())) { + throw new ResourceControlException("PID_FILE_OWNER_MISMATCH"); + } + String command = new String(Files.readAllBytes(proc.resolve("cmdline")), StandardCharsets.UTF_8) + .replace('\0', ' '); + if (expected != null && !command.contains(expected)) { + throw new ResourceControlException("PID_COMMAND_MISMATCH"); + } + + mkdir(target); + enableV2Path(backend, target); + initializeMems(backend, target); + initializeCpus(backend, target); + write(target.resolve("cgroup.procs"), String.valueOf(pid)); + if (!processIds(target.resolve("cgroup.procs")).contains(String.valueOf(pid))) { + throw new ResourceControlException("PID_PROCESS_MOVE_FAILED"); + } + return target; + } catch (NumberFormatException exception) { + throw new ResourceControlException("PID_FILE_FORMAT_INVALID"); + } catch (IOException exception) { + throw new ResourceControlException("PID_FILE_READ_FAILED:" + exception.getMessage()); + } + } + + private long processStartTimeMillis(int pid) { + long bootTimeSeconds = -1; + for (String line : read(Paths.get("/proc/stat")).split("\\R")) { + if (line.startsWith("btime ")) { + try { + bootTimeSeconds = Long.parseLong(line.substring("btime ".length()).trim()); + } catch (NumberFormatException exception) { + throw new ResourceControlException("PID_START_TIME_UNAVAILABLE"); + } + break; + } + } + String stat = read(Paths.get("/proc", String.valueOf(pid), "stat")); + int commandEnd = stat.lastIndexOf(')'); + if (bootTimeSeconds < 0 || commandEnd < 0) { + throw new ResourceControlException("PID_START_TIME_UNAVAILABLE"); + } + String[] fields = stat.substring(commandEnd + 1).trim().split("\\s+"); + if (fields.length <= 19) { + throw new ResourceControlException("PID_START_TIME_UNAVAILABLE"); + } + try { + long startTicks = Long.parseLong(fields[19]); + long clockTicks = Long.parseLong(run(null, "getconf", "CLK_TCK").trim()); + if (clockTicks <= 0) { + throw new ResourceControlException("PID_START_TIME_UNAVAILABLE"); + } + return bootTimeSeconds * 1000 + startTicks * 1000 / clockTicks; + } catch (NumberFormatException exception) { + throw new ResourceControlException("PID_START_TIME_UNAVAILABLE"); + } + } + + private String applyToGroup( + Backend backend, Path target, String desired, boolean enabled) { + enableV2Path(backend, target); + initializeMems(backend, target); + Path cpuFile = target.resolve("cpuset.cpus"); + if (!Files.isRegularFile(cpuFile)) { + throw new ResourceControlException("CPUSET_CONTROLLER_NOT_DELEGATED"); + } + String value = desired; + if (!enabled) { + if (backend.v2 && managedGroup(backend.root, target)) { + moveProcessesToParent(target); + value = ""; + } else { + value = parentCpuSet(target); + } + } + String configured = normalizeOptional(read(cpuFile)); + if (!configured.equals(value)) { + write(cpuFile, value.isEmpty() && backend.v2 ? "\n" : value); + } + if (!enabled) { + if (!normalizeOptional(read(cpuFile)).equals(value)) { + throw new ResourceControlException("CPUSET_RELEASE_MISMATCH"); + } + return ""; + } + Path effective = target.resolve("cpuset.cpus.effective"); + return normalizeOptional(read(Files.isRegularFile(effective) ? effective : cpuFile)); + } + + private MemoryResult applyMemoryLimit( + Backend backend, + MemoryBackend memoryBackend, + Path cpuTarget, + long desiredLimit) { + if (memoryBackend == null) { + throw new ResourceControlException( + "MEMORY_CONTROLLER_UNAVAILABLE"); + } + Path relative = backend.root.relativize(cpuTarget); + Path memoryTarget = controllerTarget(memoryBackend.root, relative); + boolean managed = managedGroup(backend.root, cpuTarget); + return applyMemoryTarget( + memoryBackend, memoryTarget, desiredLimit, + managed, cpuTarget); + } + + private MemoryResult applyMemoryTarget( + MemoryBackend memoryBackend, + Path memoryTarget, + long desiredLimit, + boolean managed, + Path cpuTarget) { + if (memoryBackend.v2) { + if (!Files.isDirectory(memoryTarget)) { + if (!managed || desiredLimit == 0) { + throw new ResourceControlException( + "MEMORY_CONTROLLER_UNAVAILABLE"); + } + mkdir(memoryTarget); + } + enableV2MemoryPath(memoryBackend, memoryTarget); + Path limit = memoryTarget.resolve("memory.max"); + if (!Files.isRegularFile(limit)) { + throw new ResourceControlException("MEMORY_CONTROLLER_UNAVAILABLE"); + } + if (managed && !memoryTarget.equals(cpuTarget) + && desiredLimit > 0) { + moveProcesses(cpuTarget.resolve("cgroup.procs"), + memoryTarget.resolve("cgroup.procs")); + } + String desired = desiredLimit == 0 ? "max" : String.valueOf(desiredLimit); + if (!desired.equals(read(limit).trim())) { + validateMemoryLimitAgainstUsage( + memoryTarget.resolve("memory.current"), desiredLimit); + write(limit, desired); + } + String actual = read(limit).trim(); + if (!desired.equals(actual)) { + throw new ResourceControlException("MEMORY_LIMIT_MISMATCH"); + } + if (managed && !memoryTarget.equals(cpuTarget) + && desiredLimit == 0) { + moveProcesses(memoryTarget.resolve("cgroup.procs"), + memoryTarget.getParent().resolve("cgroup.procs")); + } + return new MemoryResult(memoryBackend.name, + "max".equals(actual) ? 0L : parseMemoryLimit(actual)); + } + + Path rootLimit = memoryBackend.root.resolve("memory.limit_in_bytes"); + if (!Files.isRegularFile(rootLimit)) { + throw new ResourceControlException("MEMORY_CONTROLLER_UNAVAILABLE"); + } + if (!Files.isDirectory(memoryTarget)) { + if (!managed || desiredLimit == 0) { + throw new ResourceControlException("MEMORY_CONTROLLER_UNAVAILABLE"); + } + mkdir(memoryTarget); + } + Path limit = memoryTarget.resolve("memory.limit_in_bytes"); + if (!Files.isRegularFile(limit)) { + throw new ResourceControlException("MEMORY_CONTROLLER_UNAVAILABLE"); + } + if (managed && desiredLimit > 0) { + moveProcesses(cpuTarget.resolve("cgroup.procs"), + memoryTarget.resolve("cgroup.procs")); + } + String desired = desiredLimit == 0 + ? read(rootLimit).trim() + : String.valueOf(desiredLimit); + if (!desired.equals(read(limit).trim())) { + validateMemoryLimitAgainstUsage( + memoryTarget.resolve("memory.usage_in_bytes"), desiredLimit); + write(limit, desired); + } + if (!desired.equals(read(limit).trim())) { + throw new ResourceControlException("MEMORY_LIMIT_MISMATCH"); + } + if (managed && desiredLimit == 0) { + moveProcesses(memoryTarget.resolve("cgroup.procs"), + memoryTarget.getParent().resolve("cgroup.procs")); + } + return new MemoryResult(memoryBackend.name, + desiredLimit == 0 ? 0L : parseMemoryLimit(read(limit).trim())); + } + + private void enableV2MemoryPath( + MemoryBackend backend, Path target) { + if (Files.isRegularFile(target.resolve("memory.max"))) { + return; + } + Path current = backend.root; + for (Path part : backend.root.relativize(target)) { + Path child = current.resolve(part); + if (!Files.isRegularFile(child.resolve("memory.max"))) { + Path controllers = current.resolve("cgroup.controllers"); + Path control = current.resolve("cgroup.subtree_control"); + if (!Files.isRegularFile(controllers) + || !read(controllers).matches("(?s).*\\bmemory\\b.*") + || !Files.isRegularFile(control)) { + throw new ResourceControlException("MEMORY_CONTROLLER_UNAVAILABLE"); + } + write(control, "+memory"); + } + current = child; + } + } + + private void moveProcesses(Path source, Path destination) { + moveProcesses( + source, + destination, + "MEMORY_CONTROLLER_UNAVAILABLE", + "MEMORY_LIMIT_MISMATCH"); + } + + private void moveProcessesToParent(Path target) { + moveProcesses( + target.resolve("cgroup.procs"), + target.getParent().resolve("cgroup.procs"), + "CPUSET_CONTROLLER_NOT_DELEGATED", + "CPUSET_RELEASE_MISMATCH"); + } + + private void moveProcesses( + Path source, + Path destination, + String unavailableReason, + String mismatchReason) { + if (!Files.isRegularFile(source) || !Files.isRegularFile(destination)) { + throw new ResourceControlException(unavailableReason); + } + + for (int attempt = 0; attempt < PROCESS_MOVE_ATTEMPTS; attempt++) { + Set destinationPids = processIds(destination); + for (String pid : processIds(source)) { + if (destinationPids.contains(pid) + || !Files.isDirectory(Paths.get("/proc", pid))) { + continue; + } + try { + write(destination, pid); + } catch (ResourceControlException exception) { + if (Files.isDirectory(Paths.get("/proc", pid))) { + throw exception; + } + } + } + + destinationPids = processIds(destination); + boolean complete = true; + for (String pid : processIds(source)) { + if (Files.isDirectory(Paths.get("/proc", pid)) + && !destinationPids.contains(pid)) { + complete = false; + break; + } + } + if (complete) { + return; + } + } + throw new ResourceControlException(mismatchReason); + } + + private Set processIds(Path source) { + Set pids = new LinkedHashSet<>(); + for (String pid : read(source).trim().split("\\s+")) { + if (pid.matches("[1-9][0-9]*")) { + pids.add(pid); + } + } + return pids; + } + + private boolean managedGroup(Path root, Path target) { + Path relative = root.relativize(target); + return relative.getNameCount() > 0 + && relative.getName(0).toString().startsWith("zstack-role-"); + } + + private long parseMemoryLimit(String value) { + try { + return Long.parseLong(value); + } catch (NumberFormatException exception) { + throw new ResourceControlException("MEMORY_LIMIT_INVALID"); + } + } + + private void validateMemoryLimitAgainstUsage(Path usage, long desiredLimit) { + if (desiredLimit == 0) { + return; + } + long current = Math.max( + parseMemoryLimit(read(usage).trim()), + residentMemoryUsage(usage.getParent().resolve("cgroup.procs"))); + if (desiredLimit < current) { + throw new ResourceControlException("MEMORY_LIMIT_BELOW_CURRENT_USAGE"); + } + } + + private long residentMemoryUsage(Path processFile) { + if (!Files.isRegularFile(processFile)) { + return 0; + } + long total = 0; + for (String pid : processIds(processFile)) { + Path process = Paths.get("/proc", pid); + Path status = process.resolve("status"); + if (!Files.isRegularFile(status)) { + continue; + } + String value; + try { + value = read(status); + } catch (ResourceControlException exception) { + if (!Files.isDirectory(process)) { + continue; + } + throw exception; + } + for (String line : value.split("\\R")) { + if (!line.startsWith("VmRSS:")) { + continue; + } + String[] fields = line.trim().split("\\s+"); + if (fields.length < 2 || !fields[1].matches("[0-9]+")) { + throw new ResourceControlException("MEMORY_USAGE_INVALID"); + } + try { + total = Math.addExact( + total, Math.multiplyExact(Long.parseLong(fields[1]), 1024L)); + } catch (NumberFormatException exception) { + throw new ResourceControlException("MEMORY_USAGE_INVALID"); + } catch (ArithmeticException exception) { + return Long.MAX_VALUE; + } + break; + } + } + return total; + } + + private void validateMemoryLimit(Long value) { + if (value == null) { + return; + } + long mebibyte = SizeUnit.MEGABYTE.toByte(1); + if (value < 0 || value % mebibyte != 0) { + throw new ResourceControlException("MEMORY_LIMIT_INVALID"); + } + } + + private void enableV2Path(Backend backend, Path target) { + if (!backend.v2) { + return; + } + Path current = backend.root; + for (Path part : backend.root.relativize(target)) { + Path child = current.resolve(part); + if (!Files.isRegularFile(child.resolve("cpuset.cpus"))) { + Path control = current.resolve("cgroup.subtree_control"); + if (!Files.isRegularFile(control)) { + throw new ResourceControlException("CGROUP_V2_SUBTREE_CONTROL_MISSING"); + } + write(control, "+cpuset"); + } + current = child; + } + } + + private void initializeMems(Backend backend, Path target) { + Path mems = target.resolve("cpuset.mems"); + if (!Files.isRegularFile(mems) || !read(mems).trim().isEmpty()) { + return; + } + Path parent = target.getParent(); + Path source = parent.resolve("cpuset.mems.effective"); + if (!Files.isRegularFile(source)) { + source = parent.resolve("cpuset.mems"); + } + String value = read(source).trim(); + if (value.isEmpty()) { + throw new ResourceControlException("CPUSET_MEMS_EMPTY"); + } + write(mems, value); + } + + private void initializeCpus(Backend backend, Path target) { + if (backend.v2) { + return; + } + Path cpus = target.resolve("cpuset.cpus"); + if (read(cpus).trim().isEmpty()) { + write(cpus, parentCpuSet(target)); + } + } + + private String parentCpuSet(Path target) { + Path parent = target.getParent(); + Path source = parent.resolve("cpuset.cpus.effective"); + if (!Files.isRegularFile(source)) { + source = parent.resolve("cpuset.cpus"); + } + String value = normalizeOptional(read(source)); + if (value.isEmpty()) { + throw new ResourceControlException("PARENT_CPUSET_EMPTY"); + } + return value; + } + + private Map systemdProperties(String unit) { + String output = run(null, + "systemctl", "show", unit, + "--property=LoadState", "--property=ActiveState", + "--property=ControlGroup", "--property=MainPID"); + Map properties = new HashMap<>(); + for (String line : output.split("\\R")) { + int separator = line.indexOf('='); + if (separator > 0) { + properties.put(line.substring(0, separator), line.substring(separator + 1)); + } + } + return properties; + } + + private void mkdir(Path path) { + if (!Files.isDirectory(path)) { + run(null, "sudo", "-n", "mkdir", "-p", path.toString()); + } + run(null, "sudo", "-n", "chmod", "0755", path.toString()); + } + + private String read(Path path) { + try { + return new String(Files.readAllBytes(path), StandardCharsets.US_ASCII); + } catch (IOException exception) { + throw new ResourceControlException("READ_FAILED:" + path + ":" + exception.getMessage()); + } + } + + private void write(Path path, String value) { + run(value.getBytes(StandardCharsets.US_ASCII), + "sudo", "-n", "tee", path.toString()); + } + + private String run(byte[] input, String... command) { + if (environment.commandExecutor != null) { + return environment.commandExecutor.run(input, command); + } + Process process = null; + try { + process = new ProcessBuilder(command).redirectErrorStream(true).start(); + if (input != null) { + try (OutputStream stream = process.getOutputStream()) { + stream.write(input); + } + } else { + process.getOutputStream().close(); + } + if (!process.waitFor(COMMAND_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new ResourceControlException("COMMAND_TIMEOUT:" + command[0]); + } + String output = readStream(process.getInputStream()); + if (process.exitValue() != 0) { + throw new ResourceControlException( + "COMMAND_FAILED:" + command[0] + ":" + output.trim()); + } + return output; + } catch (IOException exception) { + throw new ResourceControlException("COMMAND_FAILED:" + command[0] + ":" + exception.getMessage()); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new ResourceControlException("COMMAND_INTERRUPTED:" + command[0]); + } finally { + if (process != null) { + process.destroy(); + } + } + } + + private String readStream(InputStream stream) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int count; + while ((count = stream.read(buffer)) != -1) { + output.write(buffer, 0, count); + } + return new String(output.toByteArray(), StandardCharsets.UTF_8); + } + + private String normalizeOptional(String value) { + String trimmed = value == null ? "" : value.trim(); + return trimmed.isEmpty() ? "" : PhysicalServerCpuSet.normalize(trimmed); + } + + private Path underRoot(Path root, Path path) { + if (!path.equals(root) && !path.startsWith(root)) { + throw new ResourceControlException("CONTROL_GROUP_PATH_INVALID"); + } + return path; + } + + private String stripRoot(String path) { + return path.startsWith("/") ? path.substring(1) : path; + } + + private String decodeMountPath(String value) { + return value.replace("\\040", " ") + .replace("\\011", "\t") + .replace("\\012", "\n") + .replace("\\134", "\\"); + } + + private String safeRole(String roleType) { + String value = roleType == null ? "" : roleType.replaceAll("[^a-zA-Z0-9_.-]", "-"); + if (value.isEmpty()) { + throw new ResourceControlException("ROLE_TYPE_INVALID"); + } + return value; + } + + private String reasonCode(ResourceControlException exception) { + String message = exception.getMessage(); + if (message == null || message.isEmpty()) { + return "RESOURCE_CONTROL_ERROR"; + } + int separator = message.indexOf(':'); + return separator > 0 ? message.substring(0, separator) : message; + } + + private static class Backend { + private final String name; + private final Path root; + private final boolean v2; + + private Backend(String name, Path root, boolean v2) { + this.name = name; + this.root = root; + this.v2 = v2; + } + } + + private static class MemoryBackend { + private final String name; + private final Path root; + private final boolean v2; + + private MemoryBackend(String name, Path root, boolean v2) { + this.name = name; + this.root = root; + this.v2 = v2; + } + } + + private static class MemoryResult { + private final String backend; + private final long actualLimit; + + private MemoryResult(String backend, long actualLimit) { + this.backend = backend; + this.actualLimit = actualLimit; + } + } + + private static class ServiceTarget { + private final String state; + private final Path path; + + private ServiceTarget(String state, Path path) { + this.state = state; + this.path = path; + } + } + + interface CommandExecutor { + String run(byte[] input, String... command); + } + + static class ExecutionEnvironment { + private final Path v2Root; + private final Path v1Root; + private final Path v1MemoryRoot; + private final Path v1SystemdRoot; + private final List v1CpuacctRoots; + private final Path procMounts; + private final Path systemdUnitRoot; + private final CommandExecutor commandExecutor; + + ExecutionEnvironment( + Path v2Root, + Path v1Root, + Path v1MemoryRoot, + Path v1SystemdRoot, + List v1CpuacctRoots, + Path procMounts, + Path systemdUnitRoot, + CommandExecutor commandExecutor) { + this.v2Root = v2Root; + this.v1Root = v1Root; + this.v1MemoryRoot = v1MemoryRoot; + this.v1SystemdRoot = v1SystemdRoot; + this.v1CpuacctRoots = new ArrayList<>(v1CpuacctRoots); + this.procMounts = procMounts; + this.systemdUnitRoot = systemdUnitRoot; + this.commandExecutor = commandExecutor; + } + + private static ExecutionEnvironment system() { + return new ExecutionEnvironment( + Paths.get("/sys/fs/cgroup"), + Paths.get("/sys/fs/cgroup/cpuset"), + Paths.get("/sys/fs/cgroup/memory"), + Paths.get("/sys/fs/cgroup/systemd"), + Arrays.asList( + Paths.get("/sys/fs/cgroup/cpu,cpuacct"), + Paths.get("/sys/fs/cgroup/cpuacct")), + Paths.get("/proc/mounts"), + Paths.get("/etc/systemd/system"), + null); + } + } + + public static class ResourceControlException extends RuntimeException { + public ResourceControlException(String message) { + super(message); + } + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodePhysicalServerAdapter.java b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodePhysicalServerAdapter.java new file mode 100644 index 00000000000..a89c718524b --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodePhysicalServerAdapter.java @@ -0,0 +1,626 @@ +package org.zstack.portal.managementnode; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.core.Platform; +import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.MessageSafe; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.AbstractService; +import org.zstack.header.Component; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.managementnode.ManagementNodeChangeListener; +import org.zstack.header.managementnode.ManagementNodeInventory; +import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; +import org.zstack.header.managementnode.ManagementNodeState; +import org.zstack.header.managementnode.ManagementNodeVO; +import org.zstack.header.managementnode.ManagementNodeVO_; +import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter; +import org.zstack.header.physicalserver.PhysicalServerResourceApplicationMode; +import org.zstack.header.physicalserver.PhysicalServerResourceConsumerState; +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode; +import org.zstack.header.physicalserver.ManagedServiceResourceUsage; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.header.physicalserver.ResourceControlCommand; +import org.zstack.header.physicalserver.ResourceControlResponse; +import org.zstack.header.physicalserver.ResourceConsumerHandle; +import org.zstack.header.physicalserver.RoleServiceManifest; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import javax.persistence.Tuple; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.zstack.core.Platform.operr; + +public class ManagementNodePhysicalServerAdapter extends AbstractService implements + PhysicalServerResourceControlAdapter, + ManagementNodeChangeListener, + ManagementNodeReadyExtensionPoint, + Component { + public static final String ROLE_TYPE = "MANAGEMENT"; + public static final String ROLE_SERVICE_MANIFEST_PATH = + "physical-server-roles/management.yaml"; + private static final String SERVICE_ID = "managementNodePhysicalServerResourceControl"; + private static final String ERROR_CODE = "ORG_ZSTACK_PORTAL_10000"; + private static final CLogger logger = Utils.getLogger( + ManagementNodePhysicalServerAdapter.class); + private static final RoleServiceManifest ROLE_SERVICES = + RoleServiceManifest.load( + ROLE_SERVICE_MANIFEST_PATH, + ROLE_TYPE, + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES); + private final AtomicReference> nodeRelations = + new AtomicReference<>(Collections.emptyMap()); + private volatile String testSerialNumber; + + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + @Autowired + private DatabaseFacade dbf; + @Autowired + private CloudBus bus; + @Autowired + private LocalCpuTopologyCollector localTopology; + @Autowired + private LocalResourceControlExecutor localResourceControlExecutor; + + @Override + @MessageSafe + public void handleMessage(Message msg) { + if (msg instanceof CollectManagementNodeCpuTopologyMsg) { + handle((CollectManagementNodeCpuTopologyMsg) msg); + } else if (msg instanceof ApplyManagementNodeResourceControlMsg) { + handle((ApplyManagementNodeResourceControlMsg) msg); + } else if (msg instanceof CollectManagementNodeManagedServicesMsg) { + handle((CollectManagementNodeManagedServicesMsg) msg); + } else if (msg instanceof RestartManagementNodeManagedServicesMsg) { + handle((RestartManagementNodeManagedServicesMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handle(CollectManagementNodeCpuTopologyMsg msg) { + CollectManagementNodeCpuTopologyReply reply = + new CollectManagementNodeCpuTopologyReply(); + if (!owns(msg.getServerUuid())) { + reply.setError(notOwner(msg.getServerUuid())); + bus.reply(msg, reply); + return; + } + collectLocalTopology(new ReturnValueCompletion(null) { + @Override + public void success(PhysicalServerCpuTopology topology) { + reply.setTopology(topology); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(ApplyManagementNodeResourceControlMsg msg) { + ApplyManagementNodeResourceControlReply reply = + new ApplyManagementNodeResourceControlReply(); + if (!owns(msg.getServerUuid())) { + reply.setError(notOwner(msg.getServerUuid())); + bus.reply(msg, reply); + return; + } + applyLocalResourceControl( + msg.getCommand(), new ReturnValueCompletion(null) { + @Override + public void success(ResourceControlResponse response) { + reply.setResponse(response); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(CollectManagementNodeManagedServicesMsg msg) { + CollectManagementNodeManagedServicesReply reply = + new CollectManagementNodeManagedServicesReply(); + if (!owns(msg.getServerUuid())) { + reply.setError(notOwner(msg.getServerUuid())); + bus.reply(msg, reply); + return; + } + try { + List services = + localResourceControlExecutor.inspect( + ROLE_TYPE, + managementHandles( + msg.isIncludeAuxiliaryServices())); + for (ManagedServiceResourceUsage service : services) { + service.setRoleType(ROLE_TYPE); + } + reply.setServices(services); + } catch (RuntimeException error) { + reply.setError(operr( + ERROR_CODE, + "MANAGED_SERVICE_QUERY_FAILED: %s", + error.getMessage())); + } + bus.reply(msg, reply); + } + + private void handle(RestartManagementNodeManagedServicesMsg msg) { + MessageReply reply = new MessageReply(); + if (!owns(msg.getServerUuid())) { + reply.setError(notOwner(msg.getServerUuid())); + bus.reply(msg, reply); + return; + } + try { + localResourceControlExecutor.restart( + ROLE_SERVICES.handlesByServiceNames( + msg.getServiceNames(), + Platform.getManagementServerId(), + Platform.getManagementServerId(), + true, + Collections.emptyMap())); + } catch (RuntimeException error) { + reply.setError(operr( + ERROR_CODE, + "MANAGED_SERVICE_RESTART_FAILED: %s", + error.getMessage())); + } + bus.reply(msg, reply); + } + + private void collectLocalTopology( + ReturnValueCompletion completion) { + try { + completion.success(localTopology.collect()); + } catch (RuntimeException error) { + completion.fail(operr(ERROR_CODE, + "CPU_TOPOLOGY_COLLECTION_FAILED: %s", error.getMessage())); + } + } + + private void applyLocalResourceControl( + ResourceControlCommand command, + ReturnValueCompletion completion) { + try { + command.setSliceName(ROLE_SERVICES.getSliceName()); + command.setHandles(managementHandles(true)); + completion.success(localResourceControlExecutor.apply(command)); + } catch (RuntimeException error) { + completion.fail(operr(ERROR_CODE, + "RESOURCE_CONTROL_APPLY_FAILED: %s", error.getMessage())); + } + } + + @Override + public String getRoleType() { + return ROLE_TYPE; + } + + @Override + public PhysicalServerResourceIsolationMode getIsolationMode() { + return PhysicalServerResourceIsolationMode.SHARED; + } + + @Override + public PhysicalServerResourceApplicationMode getApplicationMode() { + return PhysicalServerResourceApplicationMode.RESOURCE_HANDLES; + } + + @Override + public Set getAssociatedServerUuids() { + if (nodeRelations.get().isEmpty()) { + refreshNodeRelations(Collections.emptySet()); + } + return new LinkedHashSet<>(nodeRelations.get().keySet()); + } + + @Override + public void refreshAssociations() { + refreshNodeRelations(Collections.emptySet()); + } + + @Override + public PhysicalServerResourceConsumerState getState(String serverUuid) { + NodeRelation node = nodeRelation(serverUuid); + if (node == null) { + return PhysicalServerResourceConsumerState.UNAVAILABLE; + } + return node.state == ManagementNodeState.RUNNING + ? PhysicalServerResourceConsumerState.AVAILABLE + : PhysicalServerResourceConsumerState.UNAVAILABLE; + } + + @Override + public Map getStates( + Collection serverUuids) { + if (serverUuids == null || serverUuids.isEmpty()) { + return Collections.emptyMap(); + } + refreshMissingNodeRelations(serverUuids); + Map result = new LinkedHashMap<>(); + for (String serverUuid : serverUuids) { + NodeRelation relation = nodeRelations.get().get(serverUuid); + result.put(serverUuid, relation == null + ? PhysicalServerResourceConsumerState.UNAVAILABLE + : relation.state == ManagementNodeState.RUNNING + ? PhysicalServerResourceConsumerState.AVAILABLE + : PhysicalServerResourceConsumerState.UNAVAILABLE); + } + return result; + } + + @Override + public void collectTopology( + String serverUuid, + ReturnValueCompletion completion) { + String nodeUuid = nodeUuid(serverUuid, null); + if (nodeUuid == null) { + completion.fail(notOwner(serverUuid)); + return; + } + CollectManagementNodeCpuTopologyMsg msg = + new CollectManagementNodeCpuTopologyMsg(); + msg.setServerUuid(serverUuid); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeServiceIdByManagementNodeId(msg, SERVICE_ID, nodeUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + CollectManagementNodeCpuTopologyReply topologyReply = reply.castReply(); + completion.success(topologyReply.getTopology()); + } + }); + } + + @Override + public void apply( + String serverUuid, + String consumerUuid, + ResourceControlCommand command, + ReturnValueCompletion completion) { + String nodeUuid = nodeUuid(serverUuid, consumerUuid); + if (nodeUuid == null) { + completion.fail(notOwner(serverUuid)); + return; + } + ApplyManagementNodeResourceControlMsg msg = + new ApplyManagementNodeResourceControlMsg(); + msg.setServerUuid(serverUuid); + msg.setCommand(command); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeServiceIdByManagementNodeId(msg, SERVICE_ID, nodeUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + ApplyManagementNodeResourceControlReply applyReply = reply.castReply(); + completion.success(applyReply.getResponse()); + } + }); + } + + @Override + public void collectManagedServiceUsage( + String serverUuid, + boolean includeAuxiliaryServices, + ReturnValueCompletion> completion) { + String nodeUuid = nodeUuid(serverUuid, null); + if (nodeUuid == null) { + completion.success(ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE")); + return; + } + CollectManagementNodeManagedServicesMsg msg = + new CollectManagementNodeManagedServicesMsg(); + msg.setServerUuid(serverUuid); + msg.setIncludeAuxiliaryServices(includeAuxiliaryServices); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeServiceIdByManagementNodeId(msg, SERVICE_ID, nodeUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.warn(String.format( + "failed to query managed services on management node[uuid:%s]: %s", + nodeUuid, reply.getError())); + completion.success(ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE")); + return; + } + CollectManagementNodeManagedServicesReply serviceReply = + reply.castReply(); + completion.success(serviceReply.getServices() == null + ? ROLE_SERVICES.managedServiceUsages( + includeAuxiliaryServices, "UNAVAILABLE") + : serviceReply.getServices()); + } + }); + } + + @Override + public void restartManagedServices( + String serverUuid, + boolean includeAuxiliaryServices, + Collection serviceNames, + Completion completion) { + String nodeUuid = nodeUuid(serverUuid, null); + if (nodeUuid == null) { + completion.fail(notOwner(serverUuid)); + return; + } + RestartManagementNodeManagedServicesMsg msg = + new RestartManagementNodeManagedServicesMsg(); + msg.setServerUuid(serverUuid); + msg.setServiceNames(new ArrayList<>(serviceNames)); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeServiceIdByManagementNodeId(msg, SERVICE_ID, nodeUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + completion.success(); + } + }); + } + + @Override + public void nodeJoin(ManagementNodeInventory inv) { + } + + @Override + public void nodeLeft(ManagementNodeInventory inv) { + if (inv.getServerUuid() != null) { + removeNodeRelation(inv.getServerUuid()); + } + if (physicalServerManager != null && inv.getServerUuid() != null) { + physicalServerManager.reconcile(inv.getServerUuid(), true); + } + } + + @Override + public void iAmDead(ManagementNodeInventory inv) { + } + + @Override + public void iJoin(ManagementNodeInventory inv) { + associateLocalNode(inv.getUuid()); + } + + @Override + public void managementNodeReady() { + associateLocalNode(Platform.getManagementServerId()); + } + + @Transactional + public void associateLocalNode(String nodeUuid) { + if (physicalServerManager == null + || !Platform.getManagementServerId().equals(nodeUuid)) { + return; + } + String current = Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .eq(ManagementNodeVO_.uuid, nodeUuid) + .findValue(); + if (current == null) { + String serialNumber = managementServerSerialNumber(); + if (serialNumber == null) { + logger.warn(String.format( + "cannot associate management node[uuid:%s] with a physical server, " + + "reasonCode[IDENTITY_UNAVAILABLE]", nodeUuid)); + return; + } + current = physicalServerManager.resolveIdentities( + Collections.singletonList( + new PhysicalServerIdentitySpec(serialNumber, null))) + .get(serialNumber); + if (current == null || linkedNode(current, nodeUuid) != null) { + return; + } + Query update = dbf.getEntityManager().createNativeQuery( + "UPDATE IGNORE ManagementNodeVO SET serverUuid = :serverUuid " + + "WHERE uuid = :nodeUuid AND serverUuid IS NULL"); + update.setParameter("serverUuid", current); + update.setParameter("nodeUuid", nodeUuid); + update.executeUpdate(); + current = Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .eq(ManagementNodeVO_.uuid, nodeUuid) + .findValue(); + } + if (current != null) { + refreshNodeRelations(Collections.singleton(current)); + physicalServerManager.ensureResourceAssignment(current, ROLE_TYPE); + physicalServerManager.reconcile(current, true); + } + } + + private String linkedNode(String serverUuid, String excludedNodeUuid) { + return Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.uuid) + .eq(ManagementNodeVO_.serverUuid, serverUuid) + .notEq(ManagementNodeVO_.uuid, excludedNodeUuid) + .findValue(); + } + + private String nodeUuid(String serverUuid, String consumerUuid) { + NodeRelation relation = nodeRelation(serverUuid); + if (relation == null) { + return null; + } + return consumerUuid == null || consumerUuid.equals(relation.nodeUuid) + ? relation.nodeUuid : null; + } + + private NodeRelation nodeRelation(String serverUuid) { + NodeRelation relation = nodeRelations.get().get(serverUuid); + if (relation != null) { + return relation; + } + refreshNodeRelations(Collections.singleton(serverUuid)); + return nodeRelations.get().get(serverUuid); + } + + private void refreshMissingNodeRelations(Collection serverUuids) { + Set missing = new LinkedHashSet<>(serverUuids); + missing.removeAll(nodeRelations.get().keySet()); + if (!missing.isEmpty()) { + refreshNodeRelations(missing); + } + } + + private void refreshNodeRelations(Collection serverUuids) { + Q query = Q.New(ManagementNodeVO.class) + .select( + ManagementNodeVO_.serverUuid, + ManagementNodeVO_.uuid, + ManagementNodeVO_.state) + .notNull(ManagementNodeVO_.serverUuid); + boolean partial = serverUuids != null && !serverUuids.isEmpty(); + if (partial) { + query.in(ManagementNodeVO_.serverUuid, serverUuids); + } + Map loaded = new LinkedHashMap<>(); + for (Tuple node : (List) query.listTuple()) { + loaded.put( + node.get(0, String.class), + new NodeRelation( + node.get(1, String.class), + node.get(2, ManagementNodeState.class))); + } + if (!partial) { + nodeRelations.set(Collections.unmodifiableMap(loaded)); + return; + } + while (true) { + Map current = nodeRelations.get(); + Map replacement = new LinkedHashMap<>(current); + for (String serverUuid : serverUuids) { + replacement.remove(serverUuid); + } + replacement.putAll(loaded); + if (nodeRelations.compareAndSet( + current, Collections.unmodifiableMap(replacement))) { + return; + } + } + } + + private void removeNodeRelation(String serverUuid) { + while (true) { + Map current = nodeRelations.get(); + if (!current.containsKey(serverUuid)) { + return; + } + Map replacement = new LinkedHashMap<>(current); + replacement.remove(serverUuid); + if (nodeRelations.compareAndSet( + current, Collections.unmodifiableMap(replacement))) { + return; + } + } + } + + private static class NodeRelation { + private final String nodeUuid; + private final ManagementNodeState state; + + private NodeRelation( + String nodeUuid, + ManagementNodeState state) { + this.nodeUuid = nodeUuid; + this.state = state; + } + } + + private boolean owns(String serverUuid) { + return serverUuid != null && serverUuid.equals(Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .eq(ManagementNodeVO_.uuid, Platform.getManagementServerId()) + .findValue()); + } + + private ErrorCode notOwner(String serverUuid) { + return operr(ERROR_CODE, + "management node[uuid:%s] is not associated with physical server[uuid:%s]", + Platform.getManagementServerId(), serverUuid); + } + + private List managementHandles( + boolean includeAuxiliaryServices) { + String consumerKey = String.format( + "management-node:%s", Platform.getManagementServerId()); + return ROLE_SERVICES.handles( + consumerKey, + consumerKey, + includeAuxiliaryServices); + } + + private String managementServerSerialNumber() { + return CoreGlobalProperty.UNIT_TEST_ON && testSerialNumber != null + ? Platform.normalizeMachineSerialNumber(testSerialNumber) + : Platform.getManagementServerSerialNumber(); + } + + public void setTestSerialNumber(String serialNumber) { + if (!CoreGlobalProperty.UNIT_TEST_ON) { + throw new IllegalStateException( + "test serial number is only available in unit-test mode"); + } + testSerialNumber = serialNumber; + } + + @Override + public boolean start() { + return true; + } + + @Override + public boolean stop() { + return true; + } + + @Override + public String getId() { + return bus.makeLocalServiceId(SERVICE_ID); + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/RestartManagementNodeManagedServicesMsg.java b/portal/src/main/java/org/zstack/portal/managementnode/RestartManagementNodeManagedServicesMsg.java new file mode 100644 index 00000000000..1c2fb09480f --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/RestartManagementNodeManagedServicesMsg.java @@ -0,0 +1,27 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.NeedReplyMessage; + +import java.util.ArrayList; +import java.util.List; + +public class RestartManagementNodeManagedServicesMsg extends NeedReplyMessage { + private String serverUuid; + private List serviceNames = new ArrayList<>(); + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public List getServiceNames() { + return serviceNames; + } + + public void setServiceNames(List serviceNames) { + this.serviceNames = serviceNames; + } +} diff --git a/portal/src/main/resources/physical-server-roles/management.yaml b/portal/src/main/resources/physical-server-roles/management.yaml new file mode 100644 index 00000000000..3f638a28e5a --- /dev/null +++ b/portal/src/main/resources/physical-server-roles/management.yaml @@ -0,0 +1,84 @@ +roleType: MANAGEMENT +applicationMode: RESOURCE_HANDLES +sliceName: zstack-management.slice +services: + - name: management-node + handleType: SYSTEMD_UNIT + value: zstack.service + required: true + restartable: false + - name: mariadb + handleType: SYSTEMD_UNIT + value: mariadb.service + required: false + restartable: false + - name: prometheus + handleType: SYSTEMD_UNIT + value: prometheus.service + required: false + restartable: true + - name: pushgateway + handleType: SYSTEMD_UNIT + value: pushgateway.service + required: false + restartable: true + - name: zstack-ui-nginx + handleType: SYSTEMD_UNIT + value: zstack-ui-nginx.service + required: false + restartable: false + auxiliary: true + - name: zops-ui + handleType: SYSTEMD_UNIT + value: zops-ui.service + required: false + restartable: false + auxiliary: true + - name: zops + handleType: SYSTEMD_UNIT + value: zops.service + required: false + restartable: false + auxiliary: true + - name: vector + handleType: SYSTEMD_UNIT + value: vector.service + required: false + restartable: true + auxiliary: true + - name: filebeat + handleType: SYSTEMD_UNIT + value: filebeat.service + required: false + restartable: true + auxiliary: true + - name: node-exporter + handleType: SYSTEMD_UNIT + value: node_exporter.service + required: false + restartable: true + auxiliary: true + - name: collectd + handleType: SYSTEMD_UNIT + value: collectd.service + required: false + restartable: true + auxiliary: true + - name: collectd-exporter + handleType: SYSTEMD_UNIT + value: collectd_exporter.service + required: false + restartable: true + auxiliary: true + - name: process-exporter + handleType: SYSTEMD_UNIT + value: process_exporter.service + required: false + restartable: true + auxiliary: true + - name: zstack-service-exporter + handleType: SYSTEMD_UNIT + value: zstack_service_exporter.service + required: false + restartable: true + auxiliary: true diff --git a/sdk/src/main/java/SourceClassMap.java b/sdk/src/main/java/SourceClassMap.java index 29c9197c37d..0777041d982 100644 --- a/sdk/src/main/java/SourceClassMap.java +++ b/sdk/src/main/java/SourceClassMap.java @@ -693,6 +693,9 @@ public class SourceClassMap { put("org.zstack.pciDevice.virtual.vfio_mdev.MdevDeviceState", "org.zstack.sdk.MdevDeviceState"); put("org.zstack.pciDevice.virtual.vfio_mdev.MdevDeviceStatus", "org.zstack.sdk.MdevDeviceStatus"); put("org.zstack.pciDevice.virtual.vfio_mdev.MdevDeviceType", "org.zstack.sdk.MdevDeviceType"); + put("org.zstack.physicalserver.PhysicalServerInventory", "org.zstack.sdk.PhysicalServerInventory"); + put("org.zstack.physicalserver.PhysicalServerManagedServiceInventory", "org.zstack.sdk.PhysicalServerManagedServiceInventory"); + put("org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory", "org.zstack.sdk.PhysicalServerResourceAssignmentInventory"); put("org.zstack.policyRoute.PolicyRouteRuleInventory", "org.zstack.sdk.PolicyRouteRuleInventory"); put("org.zstack.policyRoute.PolicyRouteRuleProtocol", "org.zstack.sdk.PolicyRouteRuleProtocol"); put("org.zstack.policyRoute.PolicyRouteRuleSetInventory", "org.zstack.sdk.PolicyRouteRuleSetInventory"); @@ -1409,6 +1412,9 @@ public class SourceClassMap { put("org.zstack.sdk.PciDeviceVirtStatus", "org.zstack.pciDevice.virtual.PciDeviceVirtStatus"); put("org.zstack.sdk.PendingTaskInfo", "org.zstack.header.core.progress.PendingTaskInfo"); put("org.zstack.sdk.PhysicalDriveSmartSelfTestHistoryInventory", "org.zstack.storage.device.localRaid.PhysicalDriveSmartSelfTestHistoryInventory"); + put("org.zstack.sdk.PhysicalServerInventory", "org.zstack.physicalserver.PhysicalServerInventory"); + put("org.zstack.sdk.PhysicalServerManagedServiceInventory", "org.zstack.physicalserver.PhysicalServerManagedServiceInventory"); + put("org.zstack.sdk.PhysicalServerResourceAssignmentInventory", "org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory"); put("org.zstack.sdk.PhysicalSwitchInventory", "org.zstack.network.hostNetworkInterface.PhysicalSwitchInventory"); put("org.zstack.sdk.PhysicalSwitchPortInventory", "org.zstack.network.hostNetworkInterface.PhysicalSwitchPortInventory"); put("org.zstack.sdk.PluginDriverInventory", "org.zstack.header.core.external.plugin.PluginDriverInventory"); diff --git a/sdk/src/main/java/org/zstack/sdk/GetPhysicalServerManagedServicesAction.java b/sdk/src/main/java/org/zstack/sdk/GetPhysicalServerManagedServicesAction.java new file mode 100644 index 00000000000..b90f21f7619 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/GetPhysicalServerManagedServicesAction.java @@ -0,0 +1,86 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; + +public class GetPhysicalServerManagedServicesAction extends AbstractAction { + private static final HashMap parameterMap = + new HashMap<>(); + private static final HashMap nonAPIParameterMap = + new HashMap<>(); + + public static class Result { + public ErrorCode error; + public GetPhysicalServerManagedServicesResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException(String.format( + "error[code: %s, description: %s, details: %s, globalErrorCode: %s]", + error.code, error.description, error.details, + error.globalErrorCode)); + } + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, + emptyString = true, noTrim = false) + public String serverUuid; + @Param(required = false) + public java.util.List systemTags; + @Param(required = false) + public java.util.List userTags; + @Param(required = false) + public String sessionId; + @Param(required = false) + public String accessKeyId; + @Param(required = false) + public String accessKeySecret; + @Param(required = false) + public String requestIp; + + private Result makeResult(ApiResult res) { + Result result = new Result(); + if (res.error != null) { + result.error = res.error; + return result; + } + GetPhysicalServerManagedServicesResult value = res.getResult( + GetPhysicalServerManagedServicesResult.class); + result.value = value == null + ? new GetPhysicalServerManagedServicesResult() : value; + return result; + } + + public Result call() { + return makeResult(ZSClient.call(this)); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/physical-servers/{serverUuid}/managed-services"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } +} diff --git a/sdk/src/main/java/org/zstack/sdk/GetPhysicalServerManagedServicesResult.java b/sdk/src/main/java/org/zstack/sdk/GetPhysicalServerManagedServicesResult.java new file mode 100644 index 00000000000..a41f87896b2 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/GetPhysicalServerManagedServicesResult.java @@ -0,0 +1,13 @@ +package org.zstack.sdk; + +public class GetPhysicalServerManagedServicesResult { + public java.util.List services; + + public void setServices(java.util.List services) { + this.services = services; + } + + public java.util.List getServices() { + return services; + } +} diff --git a/sdk/src/main/java/org/zstack/sdk/HostInventory.java b/sdk/src/main/java/org/zstack/sdk/HostInventory.java index 1fce8a8b39d..3d584fb8815 100644 --- a/sdk/src/main/java/org/zstack/sdk/HostInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/HostInventory.java @@ -244,6 +244,14 @@ public java.lang.String getArchitecture() { return this.architecture; } + public java.lang.String serverUuid; + public void setServerUuid(java.lang.String serverUuid) { + this.serverUuid = serverUuid; + } + public java.lang.String getServerUuid() { + return this.serverUuid; + } + public java.sql.Timestamp createDate; public void setCreateDate(java.sql.Timestamp createDate) { this.createDate = createDate; diff --git a/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java b/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java index ab788e532f0..12857365c00 100644 --- a/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java @@ -36,4 +36,12 @@ public java.sql.Timestamp getHeartBeat() { return this.heartBeat; } + public java.lang.String serverUuid; + public void setServerUuid(java.lang.String serverUuid) { + this.serverUuid = serverUuid; + } + public java.lang.String getServerUuid() { + return this.serverUuid; + } + } diff --git a/sdk/src/main/java/org/zstack/sdk/PhysicalServerInventory.java b/sdk/src/main/java/org/zstack/sdk/PhysicalServerInventory.java new file mode 100644 index 00000000000..c15a11da955 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/PhysicalServerInventory.java @@ -0,0 +1,47 @@ +package org.zstack.sdk; + + + +public class PhysicalServerInventory { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String zoneUuid; + public void setZoneUuid(java.lang.String zoneUuid) { + this.zoneUuid = zoneUuid; + } + public java.lang.String getZoneUuid() { + return this.zoneUuid; + } + + public java.lang.String serialNumber; + public void setSerialNumber(java.lang.String serialNumber) { + this.serialNumber = serialNumber; + } + public java.lang.String getSerialNumber() { + return this.serialNumber; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/PhysicalServerManagedServiceInventory.java b/sdk/src/main/java/org/zstack/sdk/PhysicalServerManagedServiceInventory.java new file mode 100644 index 00000000000..6f32b936e36 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/PhysicalServerManagedServiceInventory.java @@ -0,0 +1,29 @@ +package org.zstack.sdk; + +public class PhysicalServerManagedServiceInventory { + public String roleType; + public String serviceName; + public boolean restartable; + public String state; + public String cpuSet; + public Long cpuTime; + public Long memory; + public Long memoryLimit; + + public void setRoleType(String roleType) { this.roleType = roleType; } + public String getRoleType() { return roleType; } + public void setServiceName(String serviceName) { this.serviceName = serviceName; } + public String getServiceName() { return serviceName; } + public void setRestartable(boolean restartable) { this.restartable = restartable; } + public boolean getRestartable() { return restartable; } + public void setState(String state) { this.state = state; } + public String getState() { return state; } + public void setCpuSet(String cpuSet) { this.cpuSet = cpuSet; } + public String getCpuSet() { return cpuSet; } + public void setCpuTime(Long cpuTime) { this.cpuTime = cpuTime; } + public Long getCpuTime() { return cpuTime; } + public void setMemory(Long memory) { this.memory = memory; } + public Long getMemory() { return memory; } + public void setMemoryLimit(Long memoryLimit) { this.memoryLimit = memoryLimit; } + public Long getMemoryLimit() { return memoryLimit; } +} diff --git a/sdk/src/main/java/org/zstack/sdk/PhysicalServerResourceAssignmentInventory.java b/sdk/src/main/java/org/zstack/sdk/PhysicalServerResourceAssignmentInventory.java new file mode 100644 index 00000000000..d1dce53091a --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/PhysicalServerResourceAssignmentInventory.java @@ -0,0 +1,29 @@ +package org.zstack.sdk; + +public class PhysicalServerResourceAssignmentInventory { + public java.lang.String uuid; + public java.lang.String serverUuid; + public java.lang.String roleType; + public java.lang.String cpuSet; + public java.lang.Long memory; + public java.lang.String state; + public java.sql.Timestamp createDate; + public java.sql.Timestamp lastOpDate; + + public void setUuid(java.lang.String uuid) { this.uuid = uuid; } + public java.lang.String getUuid() { return uuid; } + public void setServerUuid(java.lang.String serverUuid) { this.serverUuid = serverUuid; } + public java.lang.String getServerUuid() { return serverUuid; } + public void setRoleType(java.lang.String roleType) { this.roleType = roleType; } + public java.lang.String getRoleType() { return roleType; } + public void setCpuSet(java.lang.String cpuSet) { this.cpuSet = cpuSet; } + public java.lang.String getCpuSet() { return cpuSet; } + public void setMemory(java.lang.Long memory) { this.memory = memory; } + public java.lang.Long getMemory() { return memory; } + public void setState(java.lang.String state) { this.state = state; } + public java.lang.String getState() { return state; } + public void setCreateDate(java.sql.Timestamp createDate) { this.createDate = createDate; } + public java.sql.Timestamp getCreateDate() { return createDate; } + public void setLastOpDate(java.sql.Timestamp lastOpDate) { this.lastOpDate = lastOpDate; } + public java.sql.Timestamp getLastOpDate() { return lastOpDate; } +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerAction.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerAction.java new file mode 100644 index 00000000000..e2b00a1fcf0 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryPhysicalServerAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.QueryPhysicalServerResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.QueryPhysicalServerResult value = res.getResult(org.zstack.sdk.QueryPhysicalServerResult.class); + ret.value = value == null ? new org.zstack.sdk.QueryPhysicalServerResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/physical-servers"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentAction.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentAction.java new file mode 100644 index 00000000000..0603ecf0614 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryPhysicalServerResourceAssignmentAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult value = res.getResult(org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult.class); + ret.value = value == null ? new org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/physical-servers/resource-assignments"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentResult.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentResult.java new file mode 100644 index 00000000000..dbfa6e4dc32 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk; + + + +public class QueryPhysicalServerResourceAssignmentResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResult.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResult.java new file mode 100644 index 00000000000..33c83a32a37 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk; + + + +public class QueryPhysicalServerResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerResourceAssignmentsAction.java b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerResourceAssignmentsAction.java new file mode 100644 index 00000000000..12aaae78960 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerResourceAssignmentsAction.java @@ -0,0 +1,109 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class RefreshPhysicalServerResourceAssignmentsAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String serverUuid; + + @Param(required = false, maxLength = 64, nonempty = false, + nullElements = false, emptyString = true, noTrim = false) + public java.lang.String roleType; + + @Param(required = false, maxLength = 64, nonempty = false, + nullElements = false, emptyString = true, noTrim = false) + public java.util.List serviceNames; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsResult value = res.getResult(org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsResult.class); + ret.value = value == null ? new org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/physical-servers/{serverUuid}/resource-assignments/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "refreshPhysicalServerResourceAssignments"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerResourceAssignmentsResult.java b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerResourceAssignmentsResult.java new file mode 100644 index 00000000000..2846b7a04ff --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerResourceAssignmentsResult.java @@ -0,0 +1,7 @@ +package org.zstack.sdk; + + + +public class RefreshPhysicalServerResourceAssignmentsResult { + +} diff --git a/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerResourceAssignmentAction.java b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerResourceAssignmentAction.java new file mode 100644 index 00000000000..96c38c75657 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerResourceAssignmentAction.java @@ -0,0 +1,110 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UpdatePhysicalServerResourceAssignmentAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.UpdatePhysicalServerResourceAssignmentResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String serverUuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String roleType; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String cpuSet; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Long memory; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.UpdatePhysicalServerResourceAssignmentResult value = res.getResult(org.zstack.sdk.UpdatePhysicalServerResourceAssignmentResult.class); + ret.value = value == null ? new org.zstack.sdk.UpdatePhysicalServerResourceAssignmentResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/physical-servers/{serverUuid}/resource-assignments/{roleType}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "updatePhysicalServerResourceAssignment"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerResourceAssignmentResult.java b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerResourceAssignmentResult.java new file mode 100644 index 00000000000..e5c0fb214e2 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerResourceAssignmentResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk; + +import org.zstack.sdk.PhysicalServerResourceAssignmentInventory; + +public class UpdatePhysicalServerResourceAssignmentResult { + public PhysicalServerResourceAssignmentInventory inventory; + public void setInventory(PhysicalServerResourceAssignmentInventory inventory) { + this.inventory = inventory; + } + public PhysicalServerResourceAssignmentInventory getInventory() { + return this.inventory; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 6e222dd4959..60860dff23e 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -2203,28 +2203,49 @@ public void success(AddonInfo addonInfo) { if (param.isNewAdded()) { controller.onFirstAdditionConfigure(new NopeCompletion()); } + String oldAddonInfo = externalVO.getAddonInfo(); + boolean addonInfoChanged = addonInfo != null + && addonInfo.changed(oldAddonInfo); + Completion persist = new Completion(completion) { + @Override + public void success() { + String newAddonInfo = addonInfo == null ? null : addonInfo.serialize(); + String newUrl = StringUtils.isEmpty(self.getUrl()) + ? externalVO.getIdentity() + "://" + self.getUuid() + : self.getUrl(); + SQL.New(ExternalPrimaryStorageVO.class) + .eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) + .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfo) + .set(ExternalPrimaryStorageVO_.config, config) + .set(ExternalPrimaryStorageVO_.url, newUrl) + .update(); + externalVO.setAddonInfo(newAddonInfo); + externalVO.setConfig(config); + externalVO.setUrl(newUrl); + if (addonInfoChanged) { + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = + new ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); + evtData.setUuid(self.getUuid()); + evtData.setAddonInfo(newAddonInfo); + evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + } + controller.setTrashExpireTime( + PrimaryStorageGlobalConfig.TRASH_EXPIRATION_TIME.value(Integer.class), + new NopeCompletion()); + pingHook(completion); + } - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) - .set(ExternalPrimaryStorageVO_.config, config) - .set(ExternalPrimaryStorageVO_.url, StringUtils.isEmpty(self.getUrl()) ? - externalVO.getIdentity() + "://" + self.getUuid() : self.getUrl()) - .update(); - - if (addonInfo != null && addonInfo.changed(externalVO.getAddonInfo())) { - String newAddonInfo = addonInfo.serialize(); - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfo) - .update(); - externalVO.setAddonInfo(newAddonInfo); - // fire event when addonInfo changed - ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = new org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); - evtData.setUuid(self.getUuid()); - evtData.setAddonInfo(newAddonInfo); - evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + @Override + public void fail(ErrorCode errorCode) { + controller.syncAddonInfo(oldAddonInfo); + completion.fail(errorCode); + } + }; + if (addonInfoChanged) { + controller.beforePersistAddonInfo(addonInfo, persist); + } else { + persist.success(); } - controller.setTrashExpireTime(PrimaryStorageGlobalConfig.TRASH_EXPIRATION_TIME.value(Integer.class), new NopeCompletion()); - pingHook(completion); } @Override @@ -2250,9 +2271,28 @@ public void run(FlowTrigger trigger, Map data) { @Override public void success(PingResult ret) { if (ret.getAddonInfo() != null && ret.getAddonInfo().changed(externalVO.getAddonInfo())) { - updateAddonInfo(ret.getAddonInfo()); + String oldAddonInfo = externalVO.getAddonInfo(); + controller.beforePersistAddonInfo( + ret.getAddonInfo(), new Completion(trigger) { + @Override + public void success() { + updateAddonInfo(ret.getAddonInfo()); + completePing(ret); + } + + @Override + public void fail(ErrorCode errorCode) { + controller.syncAddonInfo(oldAddonInfo); + trigger.fail(errorCode); + } + }); + return; } + completePing(ret); + } + + private void completePing(PingResult ret) { if (!ret.isSuccess()) { trigger.fail(operr(ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10013, "ping external primary storage[%s] failed, %s", self.getUuid(), ret.getError())); return; diff --git a/test/pom.xml b/test/pom.xml index 6505ec9324c..07b6f6f7726 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -130,6 +130,11 @@ externalStorage ${project.version} + + org.zstack + physicalServer + ${project.version} + org.zstack zbs diff --git a/test/src/test/groovy/org/zstack/portal/managementnode/LocalCpuTopologyCollectorCase.groovy b/test/src/test/groovy/org/zstack/portal/managementnode/LocalCpuTopologyCollectorCase.groovy new file mode 100644 index 00000000000..636795e6822 --- /dev/null +++ b/test/src/test/groovy/org/zstack/portal/managementnode/LocalCpuTopologyCollectorCase.groovy @@ -0,0 +1,92 @@ +package org.zstack.portal.managementnode + +import org.junit.Test +import org.zstack.header.physicalserver.PhysicalServerCpuTopology + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path + +class LocalCpuTopologyCollectorCase { + @Test + void testCollectsNumaAndSmtTopologyFromSysfsFacts() { + Path root = Files.createTempDirectory("local-cpu-topology-case-") + try { + Path cpu = root.resolve("cpu") + Path node = root.resolve("node") + write(cpu.resolve("online"), "0-3\n") + write(cpu.resolve("cpu0/topology/thread_siblings_list"), "0,2\n") + write(cpu.resolve("cpu2/topology/thread_siblings_list"), "0,2\n") + write(cpu.resolve("cpu1/topology/thread_siblings_list"), "1,3\n") + write(cpu.resolve("cpu3/topology/thread_siblings_list"), "1,3\n") + write(node.resolve("node0/cpulist"), "0,2\n") + write(node.resolve("node1/cpulist"), "1,3\n") + + PhysicalServerCpuTopology topology = + new LocalCpuTopologyCollector(cpu, node).collect() + assert topology.fingerprint() == "0:0,2;1:1,3" : + "sysfs NUMA and thread-sibling facts must become stable core groups: " + + "expected=0:0,2;1:1,3 actual=${topology.fingerprint()}" + } finally { + removeTree(root) + } + } + + @Test + void testFallsBackToOneNumaNodeWhenNodeSysfsIsAbsent() { + Path root = Files.createTempDirectory("local-cpu-flat-topology-case-") + try { + Path cpu = root.resolve("cpu") + Path missingNodeRoot = root.resolve("missing-node") + write(cpu.resolve("online"), "0-1\n") + write(cpu.resolve("cpu0/topology/thread_siblings_list"), "0-1\n") + write(cpu.resolve("cpu1/topology/thread_siblings_list"), "0-1\n") + + PhysicalServerCpuTopology topology = + new LocalCpuTopologyCollector( + cpu, missingNodeRoot).collect() + assert topology.fingerprint() == "0:0-1" : + "a machine without NUMA node directories must still expose one complete topology: " + + "expected=0:0-1 actual=${topology.fingerprint()}" + } finally { + removeTree(root) + } + } + + @Test + void testRejectsAnEmptySiblingFact() { + Path root = Files.createTempDirectory("local-cpu-invalid-topology-case-") + try { + Path cpu = root.resolve("cpu") + write(cpu.resolve("online"), "0\n") + write(cpu.resolve("cpu0/topology/thread_siblings_list"), "\n") + + Throwable failure = null + try { + new LocalCpuTopologyCollector( + cpu, root.resolve("missing-node")).collect() + } catch (Throwable error) { + failure = error + } + assert failure?.message?.contains("CPU_TOPOLOGY_INVALID") : + "empty sysfs facts must fail with a typed topology reason: " + + "actual=${failure}" + } finally { + removeTree(root) + } + } + + private static void write(Path path, String value) { + Files.createDirectories(path.parent) + Files.write(path, value.getBytes(StandardCharsets.US_ASCII)) + } + + private static void removeTree(Path root) { + if (!Files.exists(root)) { + return + } + Files.walk(root).sorted(Comparator.reverseOrder()).forEach { + Files.deleteIfExists(it) + } + } +} diff --git a/test/src/test/groovy/org/zstack/portal/managementnode/LocalResourceControlExecutorCase.groovy b/test/src/test/groovy/org/zstack/portal/managementnode/LocalResourceControlExecutorCase.groovy new file mode 100644 index 00000000000..51fc3fff137 --- /dev/null +++ b/test/src/test/groovy/org/zstack/portal/managementnode/LocalResourceControlExecutorCase.groovy @@ -0,0 +1,152 @@ +package org.zstack.portal.managementnode + +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.zstack.core.CoreGlobalProperty +import org.zstack.header.physicalserver.ManagedServiceResourceUsage +import org.zstack.header.physicalserver.ResourceConsumerHandle +import org.zstack.header.physicalserver.ResourceControlCommand +import org.zstack.header.physicalserver.ResourceControlResponse +import org.zstack.utils.data.SizeUnit + +class LocalResourceControlExecutorCase { + private LocalResourceControlExecutor executor + private ResourceConsumerHandle restartable + private ResourceConsumerHandle nonRestartable + private boolean previousUnitTestOn + + @Before + void setUp() { + previousUnitTestOn = CoreGlobalProperty.UNIT_TEST_ON + CoreGlobalProperty.UNIT_TEST_ON = true + executor = new LocalResourceControlExecutor() + executor.setTestMode(true) + restartable = systemdHandle( + "prometheus", "prometheus.service", true) + nonRestartable = systemdHandle( + "management-node", "zstack-management.service", false) + } + + @After + void cleanUp() { + executor?.setTestMode(false) + CoreGlobalProperty.UNIT_TEST_ON = previousUnitTestOn + } + + @Test + void testAppliesAndInspectsCpuAndMemoryAsOneRoleBoundary() { + ResourceControlCommand command = command( + "APPLY", "3,1-2", SizeUnit.MEGABYTE.toByte(256), + [restartable, nonRestartable]) + + ResourceControlResponse response = executor.apply(command) + + assert response.cpuSet == "1-3" : + "local execution must normalize the requested CPU set: " + + "expected=1-3 actual=${response.cpuSet}" + assert response.memory == SizeUnit.MEGABYTE.toByte(256) : + "the Role memory boundary must be preserved in platform bytes: " + + "actual=${response.memory}" + assert response.coveredServiceCount == 2 && + response.expectedServiceCount == 2 : + "every manifest handle must be covered by the Role boundary: " + + "actual=${response.coveredServiceCount}/${response.expectedServiceCount}" + assert response.results*.state == ["READY", "READY"] : + "an applied local Role must report every service ready: " + + "actual=${response.results*.state}" + + List usage = executor.inspect( + "MANAGEMENT", [restartable, nonRestartable]) + assert usage*.serviceName == ["prometheus", "management-node"] : + "inspection must retain stable service identities: " + + "actual=${usage*.serviceName}" + assert usage.every { + it.state == "RUNNING" && it.cpuSet == "3,1-2" && + it.memoryLimit == SizeUnit.MEGABYTE.toByte(256) + } : "inspection must expose the last applied CPU and memory boundary: actual=${usage}" + } + + @Test + void testReleaseDisablesEveryHandleAndClearsMemoryLimit() { + ResourceControlResponse response = executor.apply(command( + "RELEASE", "0-3", SizeUnit.MEGABYTE.toByte(128), + [restartable, nonRestartable])) + + assert response.cpuSet == "" : + "release must remove the Role CPU constraint: actual=${response.cpuSet}" + assert response.memory == 0L : + "release must remove the Role memory limit: actual=${response.memory}" + assert response.results*.state == ["DISABLED", "DISABLED"] : + "release must disable every manifest handle: actual=${response.results*.state}" + } + + @Test + void testRejectsInvalidMemoryBeforeReportingSuccess() { + Throwable failure = null + try { + executor.apply(command( + "APPLY", "0-1", 1L, [restartable])) + } catch (Throwable error) { + failure = error + } + + assert failure?.message?.contains("MEMORY_LIMIT_INVALID") : + "memory must use positive MiB-aligned platform bytes: actual=${failure}" + } + + @Test + void testRestartsOnlyExplicitRestartableSystemdHandles() { + executor.restart([restartable]) + assert executor.lastTestRestartHandles*.value == + ["prometheus.service"] : + "restart must use the stable systemd unit selected by the caller: " + + "actual=${executor.lastTestRestartHandles*.value}" + + assertRestartRejected([], + "an empty service selection must never restart the whole Role") + assertRestartRejected([nonRestartable], + "a non-restartable service must remain outside Cloud restart ownership") + + ResourceConsumerHandle pidFile = new ResourceConsumerHandle( + ResourceConsumerHandle.OWNER_PID_FILE, + "/run/example.pid", "example", "example", + false, true, "example") + assertRestartRejected([pidFile], + "restart supports stable systemd units only") + } + + private void assertRestartRejected( + List handles, String intent) { + Throwable failure = null + try { + executor.restart(handles) + } catch (Throwable error) { + failure = error + } + assert failure != null : "${intent}: expected rejection" + } + + private static ResourceControlCommand command( + String operation, + String cpuSet, + Long memory, + List handles) { + ResourceControlCommand command = new ResourceControlCommand() + command.roleType = "MANAGEMENT" + command.isolationMode = "SHARED" + command.operation = operation + command.cpuSet = cpuSet + command.memory = memory + command.handles = handles + return command + } + + private static ResourceConsumerHandle systemdHandle( + String serviceName, String unit, boolean restartable) { + return new ResourceConsumerHandle( + ResourceConsumerHandle.SYSTEMD_UNIT, + unit, serviceName, serviceName, + false, restartable, null) + } +} diff --git a/test/src/test/groovy/org/zstack/portal/managementnode/LocalResourceControlExecutorFileSystemCase.groovy b/test/src/test/groovy/org/zstack/portal/managementnode/LocalResourceControlExecutorFileSystemCase.groovy new file mode 100644 index 00000000000..b911147fcef --- /dev/null +++ b/test/src/test/groovy/org/zstack/portal/managementnode/LocalResourceControlExecutorFileSystemCase.groovy @@ -0,0 +1,737 @@ +package org.zstack.portal.managementnode + +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.zstack.core.CoreGlobalProperty +import org.zstack.header.physicalserver.ManagedServiceResourceUsage +import org.zstack.header.physicalserver.ResourceConsumerHandle +import org.zstack.header.physicalserver.ResourceControlCommand +import org.zstack.header.physicalserver.ResourceControlResponse +import org.zstack.utils.data.SizeUnit + +import java.lang.management.ManagementFactory +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.nio.file.attribute.PosixFilePermission + +class LocalResourceControlExecutorFileSystemCase { + private Path temporaryRoot + private Path v2Root + private Path v1Root + private Path v1MemoryRoot + private Path v1SystemdRoot + private Path v1CpuacctRoot + private Path procMounts + private Path systemdUnitRoot + private FakeCommandExecutor commands + private LocalResourceControlExecutor executor + private boolean previousUnitTestOn + + @Before + void setUp() { + previousUnitTestOn = CoreGlobalProperty.UNIT_TEST_ON + CoreGlobalProperty.UNIT_TEST_ON = true + temporaryRoot = Files.createTempDirectory("resource-control-case-") + v2Root = temporaryRoot.resolve("cgroup2") + v1Root = temporaryRoot.resolve("cpuset") + v1MemoryRoot = temporaryRoot.resolve("memory") + v1SystemdRoot = temporaryRoot.resolve("systemd") + v1CpuacctRoot = temporaryRoot.resolve("cpuacct") + procMounts = temporaryRoot.resolve("mounts") + systemdUnitRoot = temporaryRoot.resolve("units") + [v2Root, v1Root, v1MemoryRoot, v1SystemdRoot, + v1CpuacctRoot, systemdUnitRoot].each { Files.createDirectories(it) } + put(procMounts, "") + commands = new FakeCommandExecutor(v2Root, v1Root, v1MemoryRoot) + executor = new LocalResourceControlExecutor( + new LocalResourceControlExecutor.ExecutionEnvironment( + v2Root, v1Root, v1MemoryRoot, v1SystemdRoot, + [v1CpuacctRoot], procMounts, systemdUnitRoot, + commands)) + } + + @After + void cleanUp() { + temporaryRoot?.toFile()?.deleteDir() + CoreGlobalProperty.UNIT_TEST_ON = previousUnitTestOn + } + + @Test + void testV2SystemdRoleApplyInspectRestartAndRelease() { + configureV2(true) + Path slice = configureV2SystemdRole(true) + long memory = SizeUnit.MEGABYTE.toByte(256) + + ResourceControlResponse applied = executor.apply(command( + "APPLY", "3,1-2", memory, [systemdHandle()])) + + assert applied.results*.state == ["READY"] : + "an active service already under the Role slice must be ready: ${applied.results*.state}" + assert applied.cpuSet == "1-3" && applied.memory == memory : + "the live v2 slice must expose the applied CPU and memory boundary: ${applied.cpuSet}/${applied.memory}" + assert text(slice.resolve("cpuset.cpus")) == "1-3" + assert text(slice.resolve("memory.max")) == "${memory}" + assert text(dropIn("zstack-management.slice")) == + "[Slice]\nAllowedCPUs=1-3\nMemoryMax=${memory}" + assert text(dropIn("prometheus.service")) == + "[Service]\nSlice=zstack-management.slice" + assert commands.count("systemctl", "daemon-reload") == 1 : + "one transaction must reload systemd once even when two drop-ins change" + + executor.apply(command( + "APPLY", "1-3", memory, [systemdHandle()])) + assert commands.count("systemctl", "daemon-reload") == 1 : + "an idempotent reconcile must not reload systemd" + + List usage = executor.inspect( + "MANAGEMENT", [systemdHandle()]) + assert usage.size() == 1 && usage[0].state == "RUNNING" + assert usage[0].cpuSet == "1-3" + assert usage[0].cpuTime == 123000L : + "v2 usage_usec must be exported as platform nanoseconds" + assert usage[0].memory == SizeUnit.MEGABYTE.toByte(32) + assert usage[0].memoryLimit == memory + + executor.restart([systemdHandle()]) + assert commands.count("systemctl", "stop", "prometheus.service") == 1 + assert commands.count("systemctl", "start", "prometheus.service") == 1 + + ResourceControlResponse released = executor.apply(command( + "RELEASE", "1-3", memory, [systemdHandle()])) + assert released.results*.state == ["DISABLED"] + assert released.cpuSet == "" && released.memory == 0L + assert text(slice.resolve("cpuset.cpus")) == "0-7" : + "release must restore the parent CPU boundary while the service is still running" + assert text(slice.resolve("memory.max")) == "max" + assert !Files.exists(dropIn("zstack-management.slice")) + assert !Files.exists(dropIn("prometheus.service")) + } + + @Test + void testSystemdServiceIsPendingUntilItRestartsIntoTheRoleSlice() { + configureV2(true) + configureV2SystemdRole(false) + long memory = SizeUnit.MEGABYTE.toByte(128) + + ResourceControlResponse pending = executor.apply(command( + "APPLY", "4-5", memory, [systemdHandle()])) + assert pending.results*.state == ["PENDING_RESTART"] : + "writing the drop-in must not claim that an existing process moved slices" + + Path service = v2Root.resolve( + "zstack.slice/zstack-management.slice/prometheus.service") + configureV2Group(service, "", "0", "max", SizeUnit.MEGABYTE.toByte(16)) + put(service.resolve("cpu.stat"), "usage_usec 1\n") + commands.unit("prometheus.service", true, + "/zstack.slice/zstack-management.slice/prometheus.service") + + ResourceControlResponse ready = executor.apply(command( + "APPLY", "4-5", memory, [systemdHandle()])) + assert ready.results*.state == ["READY"] : + "the next reconcile must observe the service in the configured slice after restart" + } + + @Test + void testRejectsMemoryLimitBelowCurrentSliceUsageBeforeChangingConfig() { + configureV2(true) + Path slice = configureV2SystemdRole(true) + put(slice.resolve("memory.current"), + "${SizeUnit.MEGABYTE.toByte(192)}") + + Throwable failure = capture { + executor.apply(command( + "APPLY", "0-1", SizeUnit.MEGABYTE.toByte(128), + [systemdHandle()])) + } + + assert failure?.message == "MEMORY_LIMIT_BELOW_CURRENT_USAGE" + assert !Files.exists(dropIn("zstack-management.slice")) : + "an unsafe shrink must fail before persisting the desired systemd state" + } + + @Test + void testV2CpuAndV1MemoryWorkInHybridMode() { + configureV2(false) + Path cpuSlice = configureV2SystemdRole(true, false) + configureV1MemorySystemdRole() + long memory = SizeUnit.MEGABYTE.toByte(320) + + ResourceControlResponse response = executor.apply(command( + "APPLY", "2-3", memory, [systemdHandle()])) + + assert response.results*.state == ["READY"] : + "CPU and memory controllers must be selected independently in a hybrid hierarchy" + assert text(cpuSlice.resolve("cpuset.cpus")) == "2-3" + assert text(v1MemoryRoot.resolve( + "zstack.slice/zstack-management.slice/memory.limit_in_bytes")) == + "${memory}" + assert text(dropIn("zstack-management.slice")).contains( + "AllowedCPUs=2-3") + assert text(dropIn("zstack-management.slice")).contains( + "MemoryLimit=${memory}") + + List usage = executor.inspect( + "MANAGEMENT", [systemdHandle()]) + assert usage[0].cpuSet == "2-3" + assert usage[0].memory == SizeUnit.MEGABYTE.toByte(24) + assert usage[0].memoryLimit == memory + } + + @Test + void testV1CpuFallbackMovesSystemdProcessesIntoManagedCpuset() { + configureV1Cpuset() + String pid = currentPid() + String sliceGroup = "/zstack.slice/zstack-management.slice" + String serviceGroup = "${sliceGroup}/prometheus.service" + commands.unit("zstack-management.slice", true, sliceGroup) + commands.unit("prometheus.service", true, serviceGroup, pid) + put(v1SystemdRoot.resolve(serviceGroup.substring(1)) + .resolve("cgroup.procs"), pid) + + ResourceControlResponse response = executor.apply(command( + "APPLY", "6-7", null, [systemdHandle()])) + + Path managed = v1Root.resolve( + "zstack-role-MANAGEMENT-unit-prometheus.service") + assert response.results*.state == ["READY"] + assert text(managed.resolve("cpuset.cpus")) == "6-7" + assert text(managed.resolve("cgroup.procs")).split(/\s+/).contains(pid) : + "v1 cpuset is not systemd delegated, so the stable unit cgroup must supply its process set" + put(v1CpuacctRoot.resolve(v1Root.relativize(managed)) + .resolve("cpuacct.usage"), "987654") + List usage = executor.inspect( + "MANAGEMENT", [systemdHandle()]) + assert usage[0].state == "RUNNING" + assert usage[0].cpuSet == "6-7" && usage[0].cpuTime == 987654L : + "v1 inspection must follow the managed cpuset into cpuacct" + + ResourceControlResponse released = executor.apply(command( + "RELEASE", "6-7", null, [systemdHandle()])) + assert released.results*.state == ["DISABLED"] + assert text(managed.resolve("cpuset.cpus")) == "0-7" + } + + @Test + void testOwnerPidFileHandleAppliesAndReleasesAUnifiedV2Group() { + configureV2(true) + put(v2Root.resolve("cgroup.procs"), "") + String pid = currentPid() + ResourceConsumerHandle owner = ownerHandle("owner") + long memory = memoryLimitAboveCurrentProcessUsage() + ResourceControlCommand apply = command( + "APPLY", "0", memory, [owner]) + apply.sliceName = null + + ResourceControlResponse response = executor.apply(apply) + + Path managed = v2Root.resolve("zstack-role-MANAGEMENT-owner-owner") + assert response.results*.state == ["READY"] : + "owner apply failed: ${debugState()}" + assert text(managed.resolve("cpuset.cpus")) == "0" + assert text(managed.resolve("memory.max")) == "${memory}" + assert text(managed.resolve("cgroup.procs")).split(/\s+/).contains(pid) + + ResourceControlCommand release = command( + "RELEASE", "0", memory, [owner]) + release.sliceName = null + ResourceControlResponse released = executor.apply(release) + assert released.results*.state == ["DISABLED"] + assert text(managed.resolve("cpuset.cpus")) == "" + assert text(managed.resolve("memory.max")) == "max" + assert text(v2Root.resolve("cgroup.procs")).split(/\s+/).contains(pid) : + "release must move a managed v2 process back to its parent before clearing cpuset" + } + + @Test + void testOwnerPidFileWorksWithV1CpuAndMemoryControllers() { + configureV1Cpuset() + String pid = currentPid() + ResourceConsumerHandle owner = ownerHandle("legacy-owner") + long memory = memoryLimitAboveCurrentProcessUsage() + long rootMemory = memory + SizeUnit.GIGABYTE.toByte(4) + configureV1MemoryRoot(rootMemory) + ResourceControlCommand apply = command( + "APPLY", "1", memory, [owner]) + apply.sliceName = null + + ResourceControlResponse response = executor.apply(apply) + + Path cpu = v1Root.resolve( + "zstack-role-MANAGEMENT-owner-legacy-owner") + Path memoryGroup = v1MemoryRoot.resolve( + "zstack-role-MANAGEMENT-owner-legacy-owner") + assert response.results*.state == ["READY"] : debugState() + assert text(cpu.resolve("cpuset.cpus")) == "1" + assert text(memoryGroup.resolve("memory.limit_in_bytes")) == "${memory}" + assert text(memoryGroup.resolve("cgroup.procs")) + .split(/\s+/).contains(pid) + + ResourceControlCommand release = command( + "RELEASE", "1", memory, [owner]) + release.sliceName = null + ResourceControlResponse released = executor.apply(release) + assert released.results*.state == ["DISABLED"] + assert text(cpu.resolve("cpuset.cpus")) == "0-7" + assert text(memoryGroup.resolve("memory.limit_in_bytes")) == + "${rootMemory}" + assert text(v1MemoryRoot.resolve("cgroup.procs")) + .split(/\s+/).contains(pid) + } + + @Test + void testMemoryRequestFailsWhenOnlyCpuControllerExists() { + configureV2(false) + configureV2SystemdRole(true, false) + + ResourceControlResponse withMemory = executor.apply(command( + "APPLY", "0-1", SizeUnit.MEGABYTE.toByte(128), + [systemdHandle()])) + assert withMemory.results*.state == ["ERROR"] : + "missing memory controller must not be reported as a synchronized memory limit" + + ResourceControlResponse cpuOnly = executor.apply(command( + "APPLY", "0-1", null, [systemdHandle()])) + assert cpuOnly.results*.state == ["READY"] : + "CPU assignment must remain available independently of the memory controller" + } + + @Test + void testSlicedFlowDistinguishesOptionalMissingAndRequiredInactiveServices() { + configureV2(false) + configureV2SystemdRole(true, false) + commands.missingUnit("optional.service") + commands.unit("required.service", false, + "/zstack.slice/zstack-management.slice/required.service") + + ResourceControlResponse response = executor.apply(command( + "APPLY", "0-1", null, [ + systemdHandle("optional", "optional.service", true), + systemdHandle("required", "required.service", false) + ])) + + assert response.results*.state == ["SKIPPED", "ERROR"] + assert response.expectedServiceCount == 1 && + response.coveredServiceCount == 0 + } + + @Test + void testRestartReportsMissingInactiveAndFailedUnits() { + ResourceConsumerHandle handle = systemdHandle() + commands.missingUnit(handle.value) + assert capture { executor.restart([handle]) }?.message == + "SYSTEMD_UNIT_NOT_FOUND" + + commands.unit(handle.value, false, "/system.slice/prometheus.service") + assert capture { executor.restart([handle]) }?.message == + "SYSTEMD_UNIT_NOT_ACTIVE" + + commands.unit(handle.value, true, "/system.slice/prometheus.service") + commands.failStarts.add(handle.value) + assert capture { executor.restart([handle]) }?.message == + "SYSTEMD_UNIT_RESTART_FAILED" + } + + @Test + void testOptionalMissingServiceIsSkippedButRequiredServiceFails() { + configureV2(false) + commands.missingUnit("optional.service") + commands.missingUnit("required.service") + ResourceConsumerHandle optional = systemdHandle( + "optional", "optional.service", true) + ResourceConsumerHandle required = systemdHandle( + "required", "required.service", false) + ResourceControlCommand request = command( + "APPLY", "0-1", null, [optional, required]) + request.sliceName = null + + ResourceControlResponse response = executor.apply(request) + + assert response.results*.state == ["SKIPPED", "ERROR"] + assert response.expectedServiceCount == 1 && + response.coveredServiceCount == 0 : + "optional absence must not dilute required-service coverage" + } + + @Test + void testUnavailableControllersAreObservableInsteadOfReportedReady() { + ResourceControlCommand request = command( + "APPLY", "0-1", null, [systemdHandle()]) + request.sliceName = null + + ResourceControlResponse response = executor.apply(request) + List usage = executor.inspect( + "MANAGEMENT", [systemdHandle()]) + + assert response.results*.state == ["ERROR"] + assert response.expectedServiceCount == 1 && + response.coveredServiceCount == 0 + assert usage*.state == ["UNAVAILABLE"] + } + + @Test + void testInspectionFallsBackToMainPidWhenSystemdOmitsControlGroup() { + configureV2(false) + String pid = currentPid() + String processGroup = new File("/proc/${pid}/cgroup").readLines() + .find { it.startsWith("0::") } + .substring(3) + Path target = v2Root.resolve( + processGroup.startsWith("/") + ? processGroup.substring(1) : processGroup) + configureV2Group(target, "5", "0", null, 0) + commands.unit("prometheus.service", true, "", pid) + + List usage = executor.inspect( + "MANAGEMENT", [systemdHandle()]) + + assert usage[0].state == "RUNNING" && usage[0].cpuSet == "5" : + "MainPID must keep usage observable when systemd returns no ControlGroup" + } + + private void configureV2(boolean memory) { + Files.createDirectories(v2Root) + put(v2Root.resolve("cgroup.controllers"), + memory ? "cpuset memory cpu" : "cpuset cpu") + put(v2Root.resolve("cgroup.subtree_control"), "") + put(v2Root.resolve("cpuset.cpus.effective"), "0-7") + put(v2Root.resolve("cpuset.mems.effective"), "0") + put(v2Root.resolve("cgroup.procs"), "") + if (memory) { + put(v2Root.resolve("memory.max"), "max") + put(v2Root.resolve("memory.current"), "0") + } + } + + private Path configureV2SystemdRole( + boolean serviceInSlice, boolean memory = true) { + Path parent = v2Root.resolve("zstack.slice") + configureV2Group(parent, "0-7", "0", memory ? "max" : null, 0) + Path slice = v2Root.resolve( + "zstack.slice/zstack-management.slice") + configureV2Group(slice, "0-7", "0", memory ? "max" : null, + SizeUnit.MEGABYTE.toByte(64)) + commands.unit("zstack-management.slice", true, + "/zstack.slice/zstack-management.slice") + Path service = serviceInSlice + ? slice.resolve("prometheus.service") + : v2Root.resolve("system.slice/prometheus.service") + configureV2Group(service, "", "0", memory ? "max" : null, + SizeUnit.MEGABYTE.toByte(32)) + put(service.resolve("cpu.stat"), "usage_usec 123\n") + commands.unit("prometheus.service", true, + "/${v2Root.relativize(service)}") + return slice + } + + private void configureV2Group( + Path group, String cpus, String mems, + String memoryLimit, long memoryCurrent) { + Files.createDirectories(group) + put(group.resolve("cpuset.cpus"), cpus) + put(group.resolve("cpuset.mems"), mems) + put(group.resolve("cgroup.procs"), "") + if (memoryLimit != null) { + put(group.resolve("memory.max"), memoryLimit) + put(group.resolve("memory.current"), "${memoryCurrent}") + } + } + + private void configureV1Cpuset() { + put(v1Root.resolve("cpuset.cpus"), "0-7") + put(v1Root.resolve("cpuset.mems"), "0") + put(v1Root.resolve("cgroup.procs"), "") + } + + private void configureV1MemorySystemdRole() { + long rootLimit = SizeUnit.GIGABYTE.toByte(4) + configureV1MemoryRoot(rootLimit) + Path slice = v1MemoryRoot.resolve( + "zstack.slice/zstack-management.slice") + configureV1MemoryGroup(slice, rootLimit, + SizeUnit.MEGABYTE.toByte(64)) + configureV1MemoryGroup(slice.resolve("prometheus.service"), + rootLimit, SizeUnit.MEGABYTE.toByte(24)) + } + + private void configureV1MemoryRoot(long rootLimit) { + put(v1MemoryRoot.resolve("memory.limit_in_bytes"), "${rootLimit}") + put(v1MemoryRoot.resolve("memory.usage_in_bytes"), "0") + put(v1MemoryRoot.resolve("cgroup.procs"), "") + } + + private ResourceConsumerHandle ownerHandle(String consumerKey) { + Path pidFile = temporaryRoot.resolve("${consumerKey}.pid") + put(pidFile, currentPid()) + Files.setPosixFilePermissions(pidFile, [ + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.OTHERS_READ + ] as Set) + return new ResourceConsumerHandle( + ResourceConsumerHandle.OWNER_PID_FILE, + pidFile.toString(), consumerKey, consumerKey, + false, false, null) + } + + private void configureV1MemoryGroup( + Path group, long limit, long usage) { + Files.createDirectories(group) + put(group.resolve("memory.limit_in_bytes"), "${limit}") + put(group.resolve("memory.usage_in_bytes"), "${usage}") + put(group.resolve("cgroup.procs"), "") + } + + private Path dropIn(String unit) { + return systemdUnitRoot.resolve(unit + ".d") + .resolve("50-zstack-resource-assignment.conf") + } + + private String debugState() { + List files = [] + temporaryRoot.toFile().eachFileRecurse { File file -> + if (file.isFile()) { + files.add("${temporaryRoot.relativize(file.toPath())}=${text(file.toPath())}") + } + } + return "commands=${commands.invocations}, files=${files.sort()}" + } + + private static ResourceControlCommand command( + String operation, + String cpuSet, + Long memory, + List handles) { + ResourceControlCommand command = new ResourceControlCommand() + command.roleType = "MANAGEMENT" + command.isolationMode = "SHARED" + command.operation = operation + command.cpuSet = cpuSet + command.memory = memory + command.sliceName = "zstack-management.slice" + command.handles = handles + return command + } + + private static ResourceConsumerHandle systemdHandle( + String serviceName = "prometheus", + String unit = "prometheus.service", + boolean optional = false) { + return new ResourceConsumerHandle( + ResourceConsumerHandle.SYSTEMD_UNIT, + unit, serviceName, serviceName, + optional, true, null) + } + + private static String currentPid() { + return ManagementFactory.runtimeMXBean.name.split("@")[0] + } + + private static long memoryLimitAboveCurrentProcessUsage() { + String rss = Files.readAllLines( + java.nio.file.Paths.get("/proc/self/status"), + StandardCharsets.US_ASCII).find { it.startsWith("VmRSS:") } + long current = Long.parseLong(rss.trim().split(/\s+/)[1]) * 1024L + long desired = Math.max( + SizeUnit.GIGABYTE.toByte(4), + current + SizeUnit.GIGABYTE.toByte(1)) + long mebibyte = SizeUnit.MEGABYTE.toByte(1) + return Math.floorDiv(desired + mebibyte - 1, mebibyte) * mebibyte + } + + private static Throwable capture(Closure action) { + try { + action.call() + return null + } catch (Throwable failure) { + return failure + } + } + + private static void put(Path path, String value) { + Files.createDirectories(path.parent) + Files.write(path, value.getBytes(StandardCharsets.US_ASCII)) + } + + private static String text(Path path) { + return new String(Files.readAllBytes(path), StandardCharsets.US_ASCII) + .trim() + } + + private static class FakeCommandExecutor + implements LocalResourceControlExecutor.CommandExecutor { + private final Path v2Root + private final Path v1Root + private final Path v1MemoryRoot + private final Map> units = [:] + private final List> invocations = [] + private final Set failStarts = [] as Set + + FakeCommandExecutor(Path v2Root, Path v1Root, Path v1MemoryRoot) { + this.v2Root = v2Root + this.v1Root = v1Root + this.v1MemoryRoot = v1MemoryRoot + } + + void unit( + String name, boolean active, String controlGroup, + String mainPid = "0") { + units[name] = [ + LoadState: "loaded", + ActiveState: active ? "active" : "inactive", + ControlGroup: controlGroup, + MainPID: mainPid + ] + } + + void missingUnit(String name) { + units[name] = [ + LoadState: "not-found", + ActiveState: "inactive", + ControlGroup: "", + MainPID: "0" + ] + } + + int count(String... suffix) { + List expected = suffix.toList() + return invocations.count { List command -> + List normalized = normalize(command) + normalized == expected + } + } + + @Override + String run(byte[] input, String... command) { + List raw = command.toList() + invocations.add(new ArrayList<>(raw)) + List args = normalize(raw) + if (args[0] == "systemctl") { + return systemctl(args) + } + if (args[0] == "mkdir" && args[1] == "-p") { + Path path = java.nio.file.Paths.get(args[2]) + Files.createDirectories(path) + initializeKernelFiles(path) + return "" + } + if (args[0] == "chmod") { + return "" + } + if (args[0] == "install") { + Path source = java.nio.file.Paths.get(args[-2]) + Path destination = java.nio.file.Paths.get(args[-1]) + Files.createDirectories(destination.parent) + Files.copy(source, destination, + StandardCopyOption.REPLACE_EXISTING) + return "" + } + if (args[0] == "rm" && args[1] == "-f") { + Files.deleteIfExists(java.nio.file.Paths.get(args[2])) + return "" + } + if (args[0] == "tee") { + writeKernelFile(java.nio.file.Paths.get(args[1]), input) + return input == null ? "" : + new String(input, StandardCharsets.US_ASCII) + } + if (args == ["getconf", "CLK_TCK"]) { + return "100\n" + } + throw new AssertionError("unexpected command: ${raw}") + } + + private String systemctl(List args) { + String action = args[1] + if (action == "show") { + Map properties = units[args[2]] ?: [ + LoadState: "not-found", + ActiveState: "inactive", + ControlGroup: "", + MainPID: "0" + ] + return properties.collect { key, value -> + "${key}=${value}" + }.join("\n") + "\n" + } + if (action == "daemon-reload") { + return "" + } + if (action == "start" || action == "stop") { + args.drop(2).each { String unit -> + if (!units.containsKey(unit)) { + units[unit] = [ + LoadState: "loaded", + ActiveState: "inactive", + ControlGroup: "", + MainPID: "0" + ] + } + units[unit].ActiveState = action == "start" && + !failStarts.contains(unit) ? "active" : "inactive" + } + return "" + } + throw new AssertionError("unexpected systemctl command: ${args}") + } + + private void initializeKernelFiles(Path path) { + if (path.startsWith(v2Root) && path != v2Root) { + putIfMissing(path.resolve("cpuset.cpus"), "") + putIfMissing(path.resolve("cpuset.mems"), "") + putIfMissing(path.resolve("cgroup.procs"), "") + } + if (path.startsWith(v1Root) && path != v1Root) { + putIfMissing(path.resolve("cpuset.cpus"), "") + putIfMissing(path.resolve("cpuset.mems"), "") + putIfMissing(path.resolve("cgroup.procs"), "") + } + if (path.startsWith(v1MemoryRoot) && path != v1MemoryRoot) { + putIfMissing(path.resolve("memory.limit_in_bytes"), + LocalResourceControlExecutorFileSystemCase.text( + v1MemoryRoot.resolve("memory.limit_in_bytes"))) + putIfMissing(path.resolve("memory.usage_in_bytes"), "0") + putIfMissing(path.resolve("cgroup.procs"), "") + } + } + + private void writeKernelFile(Path path, byte[] input) { + String value = input == null ? "" : + new String(input, StandardCharsets.US_ASCII) + if (path.fileName.toString() == "cgroup.subtree_control" && + value.contains("+memory")) { + path.parent.toFile().eachDir { File child -> + putIfMissing(child.toPath().resolve("memory.max"), "max") + putIfMissing(child.toPath().resolve("memory.current"), "0") + } + } + if (path.fileName.toString() == "cgroup.procs") { + Set pids = [] as LinkedHashSet + if (Files.isRegularFile(path)) { + pids.addAll(LocalResourceControlExecutorFileSystemCase + .text(path).split(/\s+/).findAll { it }) + } + pids.addAll(value.trim().split(/\s+/).findAll { it }) + LocalResourceControlExecutorFileSystemCase.put( + path, pids.join("\n")) + return + } + LocalResourceControlExecutorFileSystemCase.put(path, value) + } + + private static void putIfMissing(Path path, String value) { + if (!Files.exists(path)) { + LocalResourceControlExecutorFileSystemCase.put(path, value) + } + } + + private static List normalize(List command) { + return command.size() >= 2 && + command[0] == "sudo" && command[1] == "-n" + ? command.drop(2) : command + } + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/physicalserver/ManagementNodeResourceAssignmentCase.groovy b/test/src/test/groovy/org/zstack/test/integration/physicalserver/ManagementNodeResourceAssignmentCase.groovy new file mode 100644 index 00000000000..a19abb60357 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/physicalserver/ManagementNodeResourceAssignmentCase.groovy @@ -0,0 +1,240 @@ +package org.zstack.test.integration.physicalserver + +import org.zstack.core.Platform +import org.zstack.core.db.Q +import org.zstack.core.db.SQL +import org.zstack.header.managementnode.ManagementNodeVO +import org.zstack.header.managementnode.ManagementNodeVO_ +import org.zstack.header.physicalserver.PhysicalServerCpuTopology +import org.zstack.header.physicalserver.PhysicalServerNumaNode +import org.zstack.header.physicalserver.ResourceConsumerHandle +import org.zstack.portal.managementnode.LocalCpuTopologyCollector +import org.zstack.portal.managementnode.LocalResourceControlExecutor +import org.zstack.portal.managementnode.ManagementNodePhysicalServerAdapter +import org.zstack.sdk.PhysicalServerResourceAssignmentInventory +import org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsAction +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SpringSpec +import org.zstack.testlib.SubCase +import org.zstack.testlib.Test +import org.zstack.utils.data.SizeUnit + +import java.util.concurrent.atomic.AtomicReference + +class ManagementNodeResourceAssignmentCase extends SubCase { + static SpringSpec springSpec = Test.makeSpring { + physicalServer() + } + + EnvSpec env + ManagementNodePhysicalServerAdapter adapter + LocalCpuTopologyCollector topologyCollector + LocalResourceControlExecutor executor + String serverUuid + + @Override + void setup() { + useSpring(springSpec) + } + + @Override + void environment() { + env = makeEnv {} + } + + @Override + void clean() { + if (adapter != null) { + adapter.setTestSerialNumber(null) + } + if (topologyCollector != null) { + topologyCollector.clearTestTopology() + } + if (executor != null) { + executor.setTestMode(false) + } + env.delete() + } + + @Override + void test() { + env.create { + adapter = bean(ManagementNodePhysicalServerAdapter.class) + topologyCollector = bean(LocalCpuTopologyCollector.class) + executor = bean(LocalResourceControlExecutor.class) + adapter.setTestSerialNumber("MN-PHYSICAL-SERVER-CASE") + SQL.New("update ManagementNodeVO m set m.serverUuid = null " + + "where m.uuid = :uuid") + .param("uuid", Platform.getManagementServerId()) + .execute() + adapter.refreshAssociations() + topologyCollector.setTestTopology(topology()) + executor.setTestMode(true) + + adapter.associateLocalNode(Platform.getManagementServerId()) + waitForLocalAssociation() + + verifyAssociationAndDefaultAssignment() + verifyCpuAndMemoryPatch() + verifyManagedServiceUsageAndRestart() + } + } + + private void waitForLocalAssociation() { + AtomicReference associated = new AtomicReference<>() + retryInSecs { + associated.set(Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .eq(ManagementNodeVO_.uuid, + Platform.getManagementServerId()) + .findValue()) + assert associated.get() != null : + "local MN serial must compose ManagementNodeVO with a PhysicalServer" + } + serverUuid = associated.get() + } + + private void verifyAssociationAndDefaultAssignment() { + String targetUuid = serverUuid + def servers = queryPhysicalServer { + conditions = ["uuid=${targetUuid}"] + } + assert servers.size() == 1 : + "MN association must create exactly one PhysicalServer: " + + "serverUuid=${serverUuid} actual=${servers.size()}" + assert servers[0].serialNumber == "mn-physical-server-case" : + "MN association must persist the normalized machine serial: " + + "expected=mn-physical-server-case " + + "actual=${servers[0].serialNumber}" + assert servers[0].zoneUuid == null : + "a dedicated management node has no Zone ownership: " + + "actualZoneUuid=${servers[0].zoneUuid}" + + refreshPhysicalServerResourceAssignments { + delegate.serverUuid = targetUuid + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment() + assert current.state == "Synced" : + "MANAGEMENT default Assignment must be applied locally: " + + "expected=Synced actual=${current.state}" + assert current.cpuSet == "0-7" : + "MANAGEMENT default must use all non-exclusive CPUs: " + + "expected=0-7 actual=${current.cpuSet}" + } + + def command = executor.lastTestCommand + assert command.roleType == "MANAGEMENT" : + "local executor must receive the MANAGEMENT Role identity: " + + "actual=${command.roleType}" + assert command.sliceName == "zstack-management.slice" : + "MANAGEMENT Role must use its configured systemd slice: " + + "actual=${command.sliceName}" + assert command.handles*.consumerKey.unique() == [ + "management-node:${Platform.getManagementServerId()}" + ] : + "every management service handle must belong to the local MN: " + + "actual=${command.handles*.consumerKey.unique()}" + } + + private void verifyCpuAndMemoryPatch() { + String targetUuid = serverUuid + updatePhysicalServerResourceAssignment { + delegate.serverUuid = targetUuid + roleType = "MANAGEMENT" + cpuSet = "0-1" + memory = SizeUnit.MEGABYTE.toByte(256) + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment() + assert current.cpuSet == "0-1" : + "MANAGEMENT CPU PATCH must become the applied boundary: " + + "expected=0-1 actual=${current.cpuSet}" + assert current.memory == SizeUnit.MEGABYTE.toByte(256) : + "MANAGEMENT memory PATCH must be stored in bytes: " + + "expected=${SizeUnit.MEGABYTE.toByte(256)} " + + "actual=${current.memory}" + assert current.state == "Synced" : + "MANAGEMENT CPU and memory PATCH must reconcile locally: " + + "expected=Synced actual=${current.state}" + } + } + + private void verifyManagedServiceUsageAndRestart() { + String targetUuid = serverUuid + def services = getPhysicalServerManagedServices { + delegate.serverUuid = targetUuid + }.services + def managementNode = services.find { + it.roleType == "MANAGEMENT" && + it.serviceName == "management-node" + } + def prometheus = services.find { + it.roleType == "MANAGEMENT" && + it.serviceName == "prometheus" + } + assert managementNode?.state == "RUNNING" : + "service inventory must expose the management node process: " + + "actual=${managementNode}" + assert managementNode.cpuSet == "0-1" && + managementNode.memoryLimit == SizeUnit.MEGABYTE.toByte(256) : + "service inventory must report the current Role boundary: " + + "actualCpuSet=${managementNode.cpuSet} " + + "actualMemory=${managementNode.memoryLimit}" + assert prometheus?.restartable : + "manifest restartability must be visible to callers: " + + "service=prometheus actual=${prometheus}" + + refreshPhysicalServerResourceAssignments { + delegate.serverUuid = targetUuid + roleType = "MANAGEMENT" + serviceNames = ["prometheus"] + } + List restarted = + executor.lastTestRestartHandles + assert restarted*.serviceName == ["prometheus"] : + "targeted restart must select only prometheus: " + + "expected=[prometheus] actual=${restarted*.serviceName}" + assert restarted[0].value == "prometheus.service" : + "restart must use the stable systemd unit from the manifest: " + + "actual=${restarted[0].value}" + + RefreshPhysicalServerResourceAssignmentsAction denied = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: targetUuid, + roleType: "MANAGEMENT", + serviceNames: ["management-node"]) + def deniedResult = denied.call() + assert deniedResult.error?.details?.contains( + "SERVICE_RESTART_NOT_ALLOWED") : + "a non-restartable core service must never be restarted by API: " + + "expected=SERVICE_RESTART_NOT_ALLOWED " + + "actual=${deniedResult.error}" + } + + private PhysicalServerResourceAssignmentInventory assignment() { + List rows = + queryPhysicalServerResourceAssignment { + conditions = [ + "serverUuid=${serverUuid}", + "roleType=MANAGEMENT" + ] + } as List + assert rows.size() == 1 : + "one PhysicalServer may have only one MANAGEMENT Assignment: " + + "serverUuid=${serverUuid} actual=${rows.size()}" + return rows[0] + } + + private static PhysicalServerCpuTopology topology() { + PhysicalServerNumaNode node = new PhysicalServerNumaNode() + node.nodeId = "0" + node.onlineCpus = ["0", "1", "2", "3", "4", "5", "6", "7"] + node.coreGroups = [ + ["0", "4"], ["1", "5"], + ["2", "6"], ["3", "7"] + ] + return PhysicalServerCpuTopology.from(["0": node]) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/physicalserver/PhysicalServerResourceAssignmentCase.groovy b/test/src/test/groovy/org/zstack/test/integration/physicalserver/PhysicalServerResourceAssignmentCase.groovy new file mode 100644 index 00000000000..03311500827 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/physicalserver/PhysicalServerResourceAssignmentCase.groovy @@ -0,0 +1,1122 @@ +package org.zstack.test.integration.physicalserver + +import org.springframework.http.HttpEntity +import org.zstack.core.Platform +import org.zstack.core.componentloader.PluginRegistry +import org.zstack.core.db.SQL +import org.zstack.header.core.Completion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.host.HostNUMANode +import org.zstack.header.host.HostVO +import org.zstack.header.physicalserver.PhysicalServerManager +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec +import org.zstack.header.physicalserver.PhysicalServerCpuTopology +import org.zstack.header.physicalserver.ManagedServiceResourceUsage +import org.zstack.header.physicalserver.PhysicalServerResourceApplicationMode +import org.zstack.header.physicalserver.PhysicalServerResourceConsumerState +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode +import org.zstack.header.physicalserver.ResourceControlCommand +import org.zstack.header.physicalserver.ResourceControlResponse +import org.zstack.header.physicalserver.ResourceControlResult +import org.zstack.kvm.KVMConstant +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KvmPhysicalServerAdapter +import org.zstack.portal.managementnode.LocalCpuTopologyCollector +import org.zstack.portal.managementnode.LocalResourceControlExecutor +import org.zstack.portal.managementnode.ManagementNodePhysicalServerAdapter +import org.zstack.sdk.HostInventory +import org.zstack.sdk.PhysicalServerResourceAssignmentInventory +import org.zstack.sdk.DeleteHostAction +import org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsAction +import org.zstack.sdk.SystemTagInventory +import org.zstack.sdk.UpdatePhysicalServerResourceAssignmentAction +import org.zstack.test.integration.kvm.host.HostEnv +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SpringSpec +import org.zstack.testlib.SubCase +import org.zstack.testlib.Test +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference + +class PhysicalServerResourceAssignmentCase extends SubCase { + static SpringSpec springSpec = Test.makeSpring { + sftpBackupStorage() + localStorage() + flatNetwork() + securityGroup() + kvm() + physicalServer() + } + + EnvSpec env + PhysicalServerManager physicalServerManager + HostInventory host + HostInventory forceDeleteHost + String physicalServerUuid + String forceDeletePhysicalServerUuid + volatile boolean failResourceControl + volatile boolean mismatchResourceControl + AtomicInteger resourceControlCalls = new AtomicInteger() + AtomicInteger capacityRefreshCalls = new AtomicInteger() + AtomicReference> restartedServices = new AtomicReference<>() + AtomicReference lastResourceControlCommand = + new AtomicReference<>() + AtomicReference> inspectedServices = new AtomicReference<>() + ManagementNodePhysicalServerAdapter managementAdapter + LocalCpuTopologyCollector localTopology + LocalResourceControlExecutor localExecutor + + @Override + void setup() { + useSpring(springSpec) + } + + @Override + void environment() { + env = HostEnv.twoHostEnv() + } + + @Override + void clean() { + if (managementAdapter != null) { + managementAdapter.setTestSerialNumber(null) + } + if (localTopology != null) { + localTopology.clearTestTopology() + } + if (localExecutor != null) { + localExecutor.setTestMode(false) + } + env.delete() + } + + @Override + void test() { + env.create { + physicalServerManager = bean(PhysicalServerManager.class) + host = env.inventoryByName("kvm1") as HostInventory + forceDeleteHost = env.inventoryByName("kvm2") as HostInventory + host = associateHostThroughSerialTag( + host, "physical-server-sdk-case") + physicalServerUuid = host.serverUuid + verifyIdentityResolution() + verifyDuplicateHostSerialIsRejected() + installCpuTopologySimulator() + installResourceControlSimulator() + + verifyHeaderOnlyDynamicSharedRole() + physicalServerManager.ensureResourceAssignment( + physicalServerUuid, "COMPUTE") + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + } + verifyKvmManifestAndManagedServiceApi() + verifyInvalidUpdateDoesNotChangeAssignment() + verifyUnsyncedAndRecovery() + verifyConcurrentSparseUpdates() + verifyExclusiveAndSharedCoexistence() + verifySharedRolesAndAuxiliaryServiceOwnership() + verifyHostCascadeCleanup() + } + } + + private HostInventory associateHostThroughSerialTag( + HostInventory target, String serialNumber) { + setHostSerialTag(target, serialNumber) + bean(KvmPhysicalServerAdapter.class).associate( + org.zstack.header.host.HostInventory.valueOf( + dbFindByUuid(target.uuid, HostVO.class))) + AtomicReference associated = new AtomicReference<>() + retryInSecs { + List hosts = queryHost { + conditions = ["uuid=${target.uuid}"] + } as List + assert hosts.size() == 1 && hosts[0].serverUuid != null : + "Host serial association must persist serverUuid: " + + "hostUuid=${target.uuid} actual=${hosts*.serverUuid}" + associated.set(hosts[0]) + } + return associated.get() + } + + private void setHostSerialTag( + HostInventory target, String serialNumber) { + List tags = querySystemTag { + conditions = [ + "resourceUuid=${target.uuid}".toString(), + "tag~=systemSerialNumber::%" + ] + } as List + if (tags.isEmpty()) { + createSystemTag { + resourceUuid = target.uuid + resourceType = "HostVO" + tag = "systemSerialNumber::${serialNumber}" + } + } else { + updateSystemTag { + uuid = tags[0].uuid + tag = "systemSerialNumber::${serialNumber}" + } + } + } + + private void verifyIdentityResolution() { + Map normalized = physicalServerManager.resolveIdentities([ + new PhysicalServerIdentitySpec( + " PHYSICAL-SERVER-SDK-CASE ", host.zoneUuid), + new PhysicalServerIdentitySpec( + "physical-server-sdk-case", null) + ]) + assert normalized == [ + "physical-server-sdk-case": physicalServerUuid + ] : + "equivalent serial reports must resolve to one PhysicalServer: " + + "expected=${physicalServerUuid} actual=${normalized}" + + Map reverse = + physicalServerManager.findSerialNumbersByServerUuids([ + physicalServerUuid, Platform.getUuid() + ]) + assert reverse == [ + (physicalServerUuid): "physical-server-sdk-case" + ] : + "bulk reverse identity lookup must ignore unknown UUIDs: " + + "expected=${physicalServerUuid}:physical-server-sdk-case " + + "actual=${reverse}" + + String conflictingZone = Platform.getUuid() + assert physicalServerManager.resolveIdentities([ + new PhysicalServerIdentitySpec( + "physical-server-sdk-case", conflictingZone) + ]).isEmpty() : + "an existing PhysicalServer must never migrate across zones by observation: " + + "serverUuid=${physicalServerUuid} conflictingZone=${conflictingZone}" + + String ambiguousSerial = "ambiguous-physical-server-case" + assert physicalServerManager.resolveIdentities([ + new PhysicalServerIdentitySpec(ambiguousSerial, host.zoneUuid), + new PhysicalServerIdentitySpec(ambiguousSerial, conflictingZone) + ]).isEmpty() : + "one observation batch with conflicting zones must be rejected: " + + "serial=${ambiguousSerial}" + assert (queryPhysicalServer { + conditions = ["serialNumber=${ambiguousSerial}"] + }).isEmpty() : + "ambiguous identity input must not create a PhysicalServer row: " + + "serial=${ambiguousSerial}" + } + + private void verifyDuplicateHostSerialIsRejected() { + setHostSerialTag(forceDeleteHost, "physical-server-sdk-case") + bean(KvmPhysicalServerAdapter.class).associate( + org.zstack.header.host.HostInventory.valueOf( + dbFindByUuid(forceDeleteHost.uuid, HostVO.class))) + List hosts = queryHost { + conditions = ["uuid=${forceDeleteHost.uuid}"] + } as List + assert hosts.size() == 1 && hosts[0].serverUuid == null : + "a PhysicalServer may be composed with only one live Host: " + + "serverUuid=${physicalServerUuid} secondHost=${hosts*.serverUuid}" + } + + private void installCpuTopologySimulator() { + env.simulator(KVMConstant.KVM_HOST_NUMA_PATH) { + return validTopologyResponse() + } + env.afterSimulator(KVMConstant.KVM_HOST_CAPACITY_PATH) { + response, HttpEntity ignored -> + capacityRefreshCalls.incrementAndGet() + return response + } + } + + private KVMAgentCommands.GetHostNUMATopologyResponse validTopologyResponse() { + HostNUMANode node = new HostNUMANode() + node.nodeID = "0" + node.cpus = ["0", "1", "2", "3", "4", "5", "6", "7"] + node.onlineCpus = node.cpus + node.coreGroups = [ + ["0", "4"], + ["1", "5"], + ["2", "6"], + ["3", "7"] + ] + node.distance = ["10"] + node.free = SizeUnit.GIGABYTE.toByte(8) + node.size = SizeUnit.GIGABYTE.toByte(8) + KVMAgentCommands.GetHostNUMATopologyResponse response = + new KVMAgentCommands.GetHostNUMATopologyResponse() + response.topology = ["0": node] + return response + } + + private void installResourceControlSimulator() { + env.simulator(KvmPhysicalServerAdapter.APPLY_RESOURCE_CONTROL_PATH) { + HttpEntity entity -> resourceControlResponse(entity) + } + env.simulator(KvmPhysicalServerAdapter.GET_MANAGED_SERVICE_USAGE_PATH) { + HttpEntity entity -> + KvmPhysicalServerAdapter.ManagedServiceAgentCommand command = + JSONObjectUtil.toObject( + entity.body, + KvmPhysicalServerAdapter.ManagedServiceAgentCommand.class) + inspectedServices.set(command.handles.collect { it.serviceName }) + KvmPhysicalServerAdapter.ManagedServiceUsageAgentResponse response = + new KvmPhysicalServerAdapter.ManagedServiceUsageAgentResponse() + response.services = command.handles.collect { + managedService("COMPUTE", it.serviceName, it.restartable) + } + return response + } + env.simulator(KvmPhysicalServerAdapter.RESTART_MANAGED_SERVICES_PATH) { + HttpEntity entity -> + KvmPhysicalServerAdapter.ManagedServiceAgentCommand command = + JSONObjectUtil.toObject( + entity.body, + KvmPhysicalServerAdapter.ManagedServiceAgentCommand.class) + restartedServices.set(command.handles.collect { it.serviceName }) + return new KVMAgentCommands.AgentResponse() + } + } + + private void verifyKvmManifestAndManagedServiceApi() { + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment() + assert current.state == "Synced" : + "COMPUTE assignment must reconcile before service inspection: " + + "expected=Synced actual=${current.state}" + assert lastResourceControlCommand.get() != null : + "KVM Apply must carry Role Manifest handles: actual=no command" + } + + KvmPhysicalServerAdapter.ResourceControlAgentCommand applied = + lastResourceControlCommand.get() + assert applied.roleType == "COMPUTE" : + "KVM command must preserve Role identity: " + + "expected=COMPUTE actual=${applied.roleType}" + assert applied.sliceName == "zstack-compute.slice" : + "KVM command must use the manifest Role slice: " + + "expected=zstack-compute.slice actual=${applied.sliceName}" + assert applied.handles*.serviceName.containsAll([ + "kvmagent", "virtlogd", "node-exporter" + ]) : + "KVM command must be generated from core and auxiliary manifest services: " + + "actual=${applied.handles*.serviceName}" + assert applied.handles.every { + it.consumerKey == "host-agent:${host.uuid}" + } : + "every KVM Handle must be scoped to the associated Host consumer: " + + "expected=host-agent:${host.uuid} " + + "actual=${applied.handles*.consumerKey}" + + def services = getPhysicalServerManagedServices { + serverUuid = physicalServerUuid + }.services + assert inspectedServices.get().containsAll([ + "kvmagent", "node-exporter" + ]) : + "managed-service query must send manifest handles to KVM Agent: " + + "actual=${inspectedServices.get()}" + assert services.find { + it.roleType == "COMPUTE" && it.serviceName == "node-exporter" + }?.restartable : + "managed-service inventory must expose manifest restartability: " + + "service=node-exporter actual=${services}" + + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + roleType = "COMPUTE" + serviceNames = ["node-exporter"] + } + assert restartedServices.get() == ["node-exporter"] : + "targeted restart must send only the selected service: " + + "expected=[node-exporter] actual=${restartedServices.get()}" + + RefreshPhysicalServerResourceAssignmentsAction roleWithoutServices = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE") + def roleWithoutServicesResult = roleWithoutServices.call() + assert roleWithoutServicesResult.error?.details?.contains( + "SERVICE_NAMES_REQUIRED") : + "roleType without serviceNames must not change refresh semantics: " + + "expected=SERVICE_NAMES_REQUIRED " + + "actual=${roleWithoutServicesResult.error}" + + RefreshPhysicalServerResourceAssignmentsAction duplicateServices = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + serviceNames: ["node-exporter", "node-exporter"]) + def duplicateServicesResult = duplicateServices.call() + assert duplicateServicesResult.error?.details?.contains( + "SERVICE_NAME_SET_INVALID") : + "duplicate serviceNames must be rejected before restart: " + + "expected=SERVICE_NAME_SET_INVALID " + + "actual=${duplicateServicesResult.error}" + } + + private void verifyHeaderOnlyDynamicSharedRole() { + String imageStoreRoleType = "IMAGE_STORE" + AtomicBoolean associated = new AtomicBoolean(true) + PhysicalServerResourceControlAdapter imageStore = [ + getRoleType: { imageStoreRoleType }, + getIsolationMode: { + PhysicalServerResourceIsolationMode.SHARED + }, + getApplicationMode: { + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES + }, + getTopologyRoleType: { "COMPUTE" }, + getAssociatedServerUuids: { + associated.get() + ? [physicalServerUuid] as Set + : Collections.emptySet() + }, + getState: { String ignored -> + associated.get() + ? PhysicalServerResourceConsumerState.AVAILABLE + : PhysicalServerResourceConsumerState.MISSING + }, + getStates: { Collection serverUuids -> + serverUuids.collectEntries { + [(it): associated.get() + ? PhysicalServerResourceConsumerState.AVAILABLE + : PhysicalServerResourceConsumerState.MISSING] + } + }, + collectTopology: { String serverUuid, def completion -> + computeResourceControlAdapter().collectTopology( + serverUuid, completion) + }, + apply: { String ignoredServer, String ignoredConsumer, + ResourceControlCommand command, def completion -> + completion.success(resourceHandleResponse(command)) + }, + collectManagedServiceUsage: { + String ignoredServer, boolean ignoredAuxiliary, + def completion -> + completion.success([ + managedService( + imageStoreRoleType, "image-store-agent", true) + ]) + }, + restartManagedServices: { + String ignoredServer, boolean ignoredAuxiliary, + Collection serviceNames, def completion -> + restartedServices.set(new ArrayList<>(serviceNames)) + completion.success() + } + ] as PhysicalServerResourceControlAdapter + bean(PluginRegistry.class).defineDynamicExtension( + PhysicalServerResourceControlAdapter.class, imageStore) + + physicalServerManager.ensureResourceAssignment( + physicalServerUuid, imageStoreRoleType) + PhysicalServerResourceAssignmentInventory created = assignment(imageStoreRoleType) + assert created.cpuSet == "" + assert created.memory == null + assert created.state == "Unsynced" + + def services = getPhysicalServerManagedServices { + serverUuid = physicalServerUuid + }.services + def imageStoreService = services.find { + it.roleType == imageStoreRoleType && it.serviceName == "image-store-agent" + } + assert imageStoreService != null + assert imageStoreService.restartable + assert imageStoreService.cpuSet == "0-7" + + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + roleType = imageStoreRoleType + serviceNames = ["image-store-agent"] + } + assert restartedServices.get() == ["image-store-agent"] + + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = + assignment(imageStoreRoleType) + assert current.cpuSet == "0-7" + assert current.state == "Synced" + } + associated.set(false) + verifyReleaseDeletesAssignment(imageStoreRoleType, null) + } + + private void verifyInvalidUpdateDoesNotChangeAssignment() { + PhysicalServerResourceAssignmentInventory before = assignment() + + UpdatePhysicalServerResourceAssignmentAction invalidCpu = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + cpuSet: "999999" + ) + UpdatePhysicalServerResourceAssignmentAction.Result cpuResult = + invalidCpu.call() + assert cpuResult.error != null + assert cpuResult.error.details.contains("CPU_SET_INVALID") + + UpdatePhysicalServerResourceAssignmentAction invalidMemory = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + memory: -1L + ) + UpdatePhysicalServerResourceAssignmentAction.Result memoryResult = + invalidMemory.call() + assert memoryResult.error != null + assert memoryResult.error.details.contains("MEMORY_INVALID") + + UpdatePhysicalServerResourceAssignmentAction unalignedMemory = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + memory: 1L + ) + def unalignedMemoryResult = unalignedMemory.call() + assert unalignedMemoryResult.error?.details?.contains( + "MEMORY_INVALID") : + "memory must use the platform byte unit with 1 MiB alignment: " + + "actual=${unalignedMemoryResult.error}" + + UpdatePhysicalServerResourceAssignmentAction empty = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE" + ) + UpdatePhysicalServerResourceAssignmentAction.Result emptyResult = + empty.call() + assert emptyResult.error != null + assert emptyResult.error.details.contains( + "RESOURCE_ASSIGNMENT_UPDATE_EMPTY") + + UpdatePhysicalServerResourceAssignmentAction unsupportedRole = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "UNKNOWN_ROLE", + cpuSet: "0" + ) + assert unsupportedRole.call().error?.details?.contains( + "ROLE_TYPE_NOT_SUPPORTED") : + "an unregistered Role must not create an Assignment implicitly" + + UpdatePhysicalServerResourceAssignmentAction missingAssignment = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "IMAGE_STORE", + cpuSet: "0" + ) + assert missingAssignment.call().error?.details?.contains( + "RESOURCE_ASSIGNMENT_NOT_FOUND") : + "a registered Adapter without an owner relation must not recreate deleted configuration" + + RefreshPhysicalServerResourceAssignmentsAction invalidServiceName = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + serviceNames: ["bad service name"]) + assert invalidServiceName.call().error?.details?.contains( + "SERVICE_NAME_SET_INVALID") : + "service restart accepts stable service identities only" + + RefreshPhysicalServerResourceAssignmentsAction tooManyServices = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + serviceNames: (0..64).collect { + "service-${it}".toString() + }) + assert tooManyServices.call().error?.details?.contains( + "SERVICE_NAME_SET_INVALID") : + "one restart request must remain bounded to 64 services" + + RefreshPhysicalServerResourceAssignmentsAction undefinedService = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + serviceNames: ["undefined-service"]) + assert undefinedService.call().error?.details?.contains( + "not defined by role") : + "a syntactically valid but unconfigured service must be rejected by the Role manifest" + + env.simulator(KVMConstant.KVM_HOST_NUMA_PATH) { + HostNUMANode node = new HostNUMANode() + node.nodeID = "0" + node.cpus = ["0", "1"] + node.onlineCpus = node.cpus + node.coreGroups = [["0"]] + KVMAgentCommands.GetHostNUMATopologyResponse response = + new KVMAgentCommands.GetHostNUMATopologyResponse() + response.topology = ["0": node] + return response + } + UpdatePhysicalServerResourceAssignmentAction invalidTopology = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + cpuSet: "0") + assert invalidTopology.call().error?.details?.contains( + "CPU_TOPOLOGY_FACT_INVALID") : + "Cloud must reject an incomplete Host topology before changing the ledger" + env.simulator(KVMConstant.KVM_HOST_NUMA_PATH) { + return validTopologyResponse() + } + + PhysicalServerResourceAssignmentInventory after = assignment() + assert after.cpuSet == before.cpuSet + assert after.memory == before.memory + assert after.state == before.state + } + + private void verifyUnsyncedAndRecovery() { + failResourceControl = true + int callsBeforeFailure = resourceControlCalls.get() + PhysicalServerResourceAssignmentInventory updated = + updatePhysicalServerResourceAssignment { + serverUuid = physicalServerUuid + roleType = "COMPUTE" + cpuSet = "0-1" + memory = SizeUnit.MEGABYTE.toByte(64) + } + assert updated.cpuSet == "0-1" + assert updated.memory == SizeUnit.MEGABYTE.toByte(64) + assert updated.state == "Unsynced" + + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + } + retryInSecs { + assert resourceControlCalls.get() > callsBeforeFailure + assert assignment().state == "Unsynced" + } + + failResourceControl = false + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment() + assert current.cpuSet == "0-1" + assert current.memory == SizeUnit.MEGABYTE.toByte(64) + assert current.state == "Synced" + } + + mismatchResourceControl = true + updatePhysicalServerResourceAssignment { + serverUuid = physicalServerUuid + roleType = "COMPUTE" + memory = SizeUnit.MEGABYTE.toByte(80) + } + retryInSecs { + assert assignment().state == "Unsynced" : + "an HTTP-success response with incomplete service coverage must not be trusted" + } + mismatchResourceControl = false + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + } + retryInSecs { + assert assignment().state == "Synced" : + "the same Assignment must recover after complete coverage is observed" + } + } + + private void verifyConcurrentSparseUpdates() { + CountDownLatch start = new CountDownLatch(1) + CountDownLatch finished = new CountDownLatch(2) + List failures = Collections.synchronizedList([]) + String sessionUuid = adminSession() + + Thread.start { + try { + start.await() + UpdatePhysicalServerResourceAssignmentAction.Result result = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: sessionUuid, + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + cpuSet: "0-2" + ).call() + assert result.error == null : result.error + } catch (Throwable error) { + failures.add(error) + } finally { + finished.countDown() + } + } + Thread.start { + try { + start.await() + UpdatePhysicalServerResourceAssignmentAction.Result result = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: sessionUuid, + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + memory: SizeUnit.MEGABYTE.toByte(96) + ).call() + assert result.error == null : result.error + } catch (Throwable error) { + failures.add(error) + } finally { + finished.countDown() + } + } + + start.countDown() + assert finished.await(30, TimeUnit.SECONDS) + assert failures.isEmpty() : failures + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment() + assert current.cpuSet == "0-2" : + "memory PATCH must preserve the concurrent CPU PATCH" + assert current.memory == SizeUnit.MEGABYTE.toByte(96) : + "CPU PATCH must preserve the concurrent memory PATCH" + assert current.state == "Synced" + } + + updatePhysicalServerResourceAssignment { + serverUuid = physicalServerUuid + roleType = "COMPUTE" + cpuSet = "0-1" + memory = SizeUnit.MEGABYTE.toByte(64) + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment() + assert current.cpuSet == "0-1" + assert current.memory == SizeUnit.MEGABYTE.toByte(64) + assert current.state == "Synced" + } + } + + private void verifyExclusiveAndSharedCoexistence() { + String storageRoleType = "TEST_STORAGE" + int capacityCallsBeforeExclusive = capacityRefreshCalls.get() + PhysicalServerResourceControlAdapter exclusiveStorage = [ + getRoleType: { storageRoleType }, + getIsolationMode: { + PhysicalServerResourceIsolationMode.EXCLUSIVE + }, + getApplicationMode: { + PhysicalServerResourceApplicationMode.PROVIDER_MANAGED + }, + getTopologyRoleType: { "COMPUTE" }, + getAssociatedServerUuids: { + [physicalServerUuid] as Set + }, + getState: { String ignored -> + PhysicalServerResourceConsumerState.AVAILABLE + }, + getStates: { Collection serverUuids -> + serverUuids.collectEntries { + [(it): PhysicalServerResourceConsumerState.AVAILABLE] + } + }, + collectTopology: { String serverUuid, def completion -> + computeResourceControlAdapter().collectTopology( + serverUuid, completion) + }, + apply: { String ignoredServer, String ignoredConsumer, + ResourceControlCommand command, def completion -> + ResourceControlResponse response = + new ResourceControlResponse() + boolean release = command.operation == "RELEASE" + response.state = release ? "DISABLED" : "READY" + response.cpuSet = release ? "" : command.cpuSet + response.memory = release ? null : command.memory + completion.success(response) + }, + collectManagedServiceUsage: { + String ignoredServer, boolean ignoredAuxiliary, + def completion -> + completion.success([ + managedService( + storageRoleType, + "test-storage", false) + ]) + }, + restartManagedServices: { + String ignoredServer, boolean ignoredAuxiliary, + Collection ignoredServices, def completion -> + completion.success() + } + ] as PhysicalServerResourceControlAdapter + bean(PluginRegistry.class).defineDynamicExtension( + PhysicalServerResourceControlAdapter.class, exclusiveStorage) + physicalServerManager.ensureResourceAssignment( + physicalServerUuid, storageRoleType) + + def service = getPhysicalServerManagedServices { + serverUuid = physicalServerUuid + }.services.find { + it.roleType == storageRoleType + } + assert service != null : + "provider-managed Role must expose its managed service: " + + "roleType=${storageRoleType} actual=no service" + assert !service.restartable : + "provider-managed service restart must remain owned by Provider: " + + "roleType=${storageRoleType} actualRestartable=${service.restartable}" + + RefreshPhysicalServerResourceAssignmentsAction providerRestart = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: storageRoleType, + serviceNames: ["test-storage"]) + def providerRestartResult = providerRestart.call() + assert providerRestartResult.error?.details?.contains( + "SERVICE_RESTART_NOT_SUPPORTED") : + "provider-managed Role must reject Cloud service restart: " + + "expected=SERVICE_RESTART_NOT_SUPPORTED " + + "actual=${providerRestartResult.error}" + + UpdatePhysicalServerResourceAssignmentAction splitCore = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: storageRoleType, + cpuSet: "2") + assert splitCore.call().error?.details?.contains( + "CPU_SIBLING_SPLIT") : + "an exclusive Role must reserve every sibling of a physical core" + + UpdatePhysicalServerResourceAssignmentAction cpuZero = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: storageRoleType, + cpuSet: "0,4") + assert cpuZero.call().error?.details?.contains( + "CPU_ZERO_RESERVED") : + "CPU0 must remain outside every exclusive service boundary" + + updatePhysicalServerResourceAssignment { + serverUuid = physicalServerUuid + roleType = storageRoleType + cpuSet = "2-3,6-7" + memory = SizeUnit.MEGABYTE.toByte(128) + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = + assignment(storageRoleType) + assert current.cpuSet == "2-3,6-7" + assert current.memory == SizeUnit.MEGABYTE.toByte(128) + assert current.state == "Synced" + } + retryInSecs { + assert capacityRefreshCalls.get() > capacityCallsBeforeExclusive : + "a newly Synced exclusive boundary must trigger Host capacity recomputation" + } + + PhysicalServerResourceAssignmentInventory before = assignment() + UpdatePhysicalServerResourceAssignmentAction overlap = + new UpdatePhysicalServerResourceAssignmentAction( + sessionId: adminSession(), + serverUuid: physicalServerUuid, + roleType: "COMPUTE", + cpuSet: "0-3" + ) + UpdatePhysicalServerResourceAssignmentAction.Result overlapResult = + overlap.call() + assert overlapResult.error != null + assert overlapResult.error.details.contains("CPU_SET_CONFLICT") + assert assignment().cpuSet == before.cpuSet + + updatePhysicalServerResourceAssignment { + serverUuid = physicalServerUuid + roleType = "COMPUTE" + cpuSet = "0-1,4-5" + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment() + assert current.cpuSet == "0-1,4-5" + assert current.memory == SizeUnit.MEGABYTE.toByte(64) + assert current.state == "Synced" + } + + int capacityCallsBeforeRelease = capacityRefreshCalls.get() + verifyReleaseDeletesAssignment(storageRoleType, null) + retryInSecs { + assert capacityRefreshCalls.get() > capacityCallsBeforeRelease : + "releasing an exclusive boundary must restore Host capacity immediately" + } + } + + private void verifySharedRolesAndAuxiliaryServiceOwnership() { + managementAdapter = bean(ManagementNodePhysicalServerAdapter.class) + localTopology = bean(LocalCpuTopologyCollector.class) + localExecutor = bean(LocalResourceControlExecutor.class) + managementAdapter.setTestSerialNumber("physical-server-sdk-case") + SQL.New("update ManagementNodeVO m set m.serverUuid = null " + + "where m.uuid = :uuid") + .param("uuid", Platform.getManagementServerId()) + .execute() + managementAdapter.refreshAssociations() + localTopology.setTestTopology(PhysicalServerCpuTopology.from([ + "0": new org.zstack.header.physicalserver.PhysicalServerNumaNode( + nodeId: "0", + onlineCpus: ["0", "1", "2", "3", "4", "5", "6", "7"], + coreGroups: [ + ["0", "4"], ["1", "5"], + ["2", "6"], ["3", "7"] + ]) + ])) + localExecutor.setTestMode(true) + managementAdapter.associateLocalNode(Platform.getManagementServerId()) + + refreshPhysicalServerResourceAssignments { + serverUuid = physicalServerUuid + } + retryInSecs { + assert assignment(physicalServerUuid, "MANAGEMENT").state == + "Synced" : + "MANAGEMENT and COMPUTE shared Roles must coexist on one PhysicalServer" + } + def services = getPhysicalServerManagedServices { + serverUuid = physicalServerUuid + }.services + assert services.find { + it.roleType == "MANAGEMENT" && + it.serviceName == "node-exporter" + } != null : + "auxiliary services must be owned by MANAGEMENT when that Role exists" + assert services.find { + it.roleType == "COMPUTE" && + it.serviceName == "node-exporter" + } == null : + "COMPUTE must not duplicate an auxiliary service already owned by MANAGEMENT" + assert !inspectedServices.get().contains("node-exporter") : + "KVM inspection command must omit auxiliary handles when MANAGEMENT owns them: " + + "actual=${inspectedServices.get()}" + } + + private void verifyHostCascadeCleanup() { + failResourceControl = true + DeleteHostAction.Result blocked = new DeleteHostAction( + sessionId: adminSession(), + uuid: host.uuid, + deleteMode: "Permissive").call() + assert blocked.error != null : + "normal Host deletion must stop when resource release fails: " + + "hostUuid=${host.uuid} actual=success" + assert queryHost { + conditions = ["uuid=${host.uuid}"] + }.size() == 1 : + "blocked Host deletion must preserve the Host relation: " + + "hostUuid=${host.uuid} actual=missing" + assert assignment().state == "Unsynced" : + "failed cascade release must preserve an Unsynced ledger for retry: " + + "expected=Unsynced actual=${assignment().state}" + + failResourceControl = false + DeleteHostAction.Result deleted = new DeleteHostAction( + sessionId: adminSession(), + uuid: host.uuid, + deleteMode: "Permissive").call() + assert deleted.error == null : + "normal Host deletion must continue after release succeeds: " + + "hostUuid=${host.uuid} actual=${deleted.error}" + retryInSecs { + assert (queryHost { + conditions = ["uuid=${host.uuid}"] + }).isEmpty() : + "successful Host cascade must delete the Host: " + + "hostUuid=${host.uuid} actual=still present" + assert assignments(physicalServerUuid, "COMPUTE").isEmpty() : + "successful Host cascade must delete the COMPUTE Assignment: " + + "serverUuid=${physicalServerUuid} actual=${assignments(physicalServerUuid, 'COMPUTE')}" + } + + forceDeleteHost = associateHostThroughSerialTag( + forceDeleteHost, "physical-server-force-delete-case") + forceDeletePhysicalServerUuid = forceDeleteHost.serverUuid + physicalServerManager.ensureResourceAssignment( + forceDeletePhysicalServerUuid, "COMPUTE") + refreshPhysicalServerResourceAssignments { + serverUuid = forceDeletePhysicalServerUuid + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment( + forceDeletePhysicalServerUuid, "COMPUTE") + assert current.state == "Synced" : + "force-delete scenario requires a synced Assignment first: " + + "expected=Synced actual=${current.state}" + } + + failResourceControl = true + DeleteHostAction.Result forceDeleted = new DeleteHostAction( + sessionId: adminSession(), + uuid: forceDeleteHost.uuid, + deleteMode: "Enforcing").call() + assert forceDeleted.error == null : + "enforcing Host deletion must not be blocked by an unreachable executor: " + + "hostUuid=${forceDeleteHost.uuid} actual=${forceDeleted.error}" + retryInSecs { + assert (queryHost { + conditions = ["uuid=${forceDeleteHost.uuid}"] + }).isEmpty() : + "enforcing cascade must delete the Host: " + + "hostUuid=${forceDeleteHost.uuid} actual=still present" + assert assignments( + forceDeletePhysicalServerUuid, "COMPUTE").isEmpty() : + "Role removal must not retain configuration after enforcing deletion: " + + "serverUuid=${forceDeletePhysicalServerUuid} " + + "actual=${assignments(forceDeletePhysicalServerUuid, 'COMPUTE')}" + } + failResourceControl = false + } + + private void verifyReleaseDeletesAssignment( + String roleType, String consumerUuid) { + CountDownLatch released = new CountDownLatch(1) + AtomicReference failure = new AtomicReference<>() + physicalServerManager.releaseResourceAssignment( + physicalServerUuid, roleType, consumerUuid, + new Completion(null) { + @Override + void success() { + released.countDown() + } + + @Override + void fail(ErrorCode errorCode) { + failure.set(errorCode) + released.countDown() + } + }) + + assert released.await(30, TimeUnit.SECONDS) + assert failure.get() == null + List assignments = + queryPhysicalServerResourceAssignment { + conditions = [ + "serverUuid=${physicalServerUuid}", + "roleType=${roleType}" + ] + } + assert assignments.isEmpty() + } + + private PhysicalServerResourceAssignmentInventory assignment() { + return assignment("COMPUTE") + } + + private PhysicalServerResourceAssignmentInventory assignment( + String roleType) { + return assignment(physicalServerUuid, roleType) + } + + private PhysicalServerResourceAssignmentInventory assignment( + String serverUuid, String roleType) { + List current = + assignments(serverUuid, roleType) + assert current.size() == 1 : + "exactly one Assignment must exist for serverUuid and roleType: " + + "serverUuid=${serverUuid} roleType=${roleType} " + + "actualCount=${current.size()}" + return current[0] + } + + private List assignments( + String serverUuid, String roleType) { + return queryPhysicalServerResourceAssignment { + conditions = [ + "serverUuid=${serverUuid}", + "roleType=${roleType}" + ] + } as List + } + + private PhysicalServerResourceControlAdapter computeResourceControlAdapter() { + return bean(PluginRegistry.class).getExtensionList( + PhysicalServerResourceControlAdapter.class).find { + it.roleType == "COMPUTE" + } + } + + private KvmPhysicalServerAdapter.ResourceControlAgentResponse resourceControlResponse( + HttpEntity entity) { + KvmPhysicalServerAdapter.ResourceControlAgentCommand command = + JSONObjectUtil.toObject( + entity.body, + KvmPhysicalServerAdapter.ResourceControlAgentCommand.class) + lastResourceControlCommand.set(command) + KvmPhysicalServerAdapter.ResourceControlAgentResponse response = + new KvmPhysicalServerAdapter.ResourceControlAgentResponse() + resourceControlCalls.incrementAndGet() + if (failResourceControl) { + response.error = "simulated resource control failure" + return response + } + + boolean release = command.operation == "RELEASE" + response.cpuSet = release ? "" : command.cpuSet + response.memory = command.memory == null + ? null : release ? 0L : command.memory + response.expectedServiceCount = mismatchResourceControl ? 2 : 1 + response.coveredServiceCount = 1 + ResourceControlResult result = new ResourceControlResult() + result.state = release ? "DISABLED" : "READY" + result.cpuSet = response.cpuSet + result.memory = response.memory + response.results = [result] + return response + } + + private ResourceControlResponse resourceHandleResponse( + ResourceControlCommand command) { + boolean release = command.operation == "RELEASE" + ResourceControlResponse response = new ResourceControlResponse() + response.cpuSet = release ? "" : command.cpuSet + response.memory = command.memory == null + ? null : release ? 0L : command.memory + response.expectedServiceCount = 1 + response.coveredServiceCount = 1 + ResourceControlResult result = new ResourceControlResult() + result.state = release ? "DISABLED" : "READY" + result.cpuSet = response.cpuSet + result.memory = response.memory + response.results = [result] + return response + } + + private ManagedServiceResourceUsage managedService( + String roleType, String serviceName, boolean restartable) { + ManagedServiceResourceUsage usage = new ManagedServiceResourceUsage() + usage.roleType = roleType + usage.serviceName = serviceName + usage.restartable = restartable + usage.state = "RUNNING" + usage.cpuSet = "0-7" + usage.cpuTime = 1000L + usage.memory = SizeUnit.MEGABYTE.toByte(96) + usage.memoryLimit = 0L + return usage + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/physicalserver/ZbsResourceAssignmentCase.groovy b/test/src/test/groovy/org/zstack/test/integration/physicalserver/ZbsResourceAssignmentCase.groovy new file mode 100644 index 00000000000..20374df3f6d --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/physicalserver/ZbsResourceAssignmentCase.groovy @@ -0,0 +1,645 @@ +package org.zstack.test.integration.physicalserver + +import org.springframework.http.HttpEntity +import org.zstack.core.componentloader.PluginRegistry +import org.zstack.core.db.Q +import org.zstack.header.core.Completion +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.host.HostNUMANode +import org.zstack.header.host.HostVO +import org.zstack.header.physicalserver.ResourceControlCommand +import org.zstack.header.physicalserver.ResourceControlResponse +import org.zstack.header.physicalserver.ResourceControlResult +import org.zstack.kvm.KVMConstant +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KvmPhysicalServerAdapter +import org.zstack.sdk.DeletePrimaryStorageAction +import org.zstack.sdk.HostInventory +import org.zstack.sdk.PhysicalServerResourceAssignmentInventory +import org.zstack.sdk.PrimaryStorageInventory +import org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsAction +import org.zstack.sdk.SystemTagInventory +import org.zstack.storage.zbs.AddonInfo +import org.zstack.storage.zbs.FakeZbsCpuIsolationProvider +import org.zstack.storage.zbs.LogicalPoolInfo +import org.zstack.storage.zbs.MdsInfo +import org.zstack.storage.zbs.ZbsCpuIsolationFact +import org.zstack.storage.zbs.ZbsCpuIsolationProvider +import org.zstack.storage.zbs.ZbsNodeRefContributor +import org.zstack.storage.zbs.ZbsNodeRefContributorImpl +import org.zstack.storage.zbs.ZbsPrimaryStorageMdsBase +import org.zstack.storage.zbs.ZbsResourceAssignmentBackend +import org.zstack.storage.zbs.ZbsResourceAssignmentGlobalConfig +import org.zstack.storage.zbs.ZbsStorageController +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_ +import org.zstack.test.integration.kvm.host.HostEnv +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SpringSpec +import org.zstack.testlib.SubCase +import org.zstack.testlib.Test +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +class ZbsResourceAssignmentCase extends SubCase { + static SpringSpec springSpec = Test.makeSpring { + sftpBackupStorage() + localStorage() + flatNetwork() + securityGroup() + kvm() + externalPrimaryStorage() + zbs() + physicalServer() + } + + static final String SERIAL = "zbs-hci-physical-server-case" + + EnvSpec env + HostInventory host + String serverUuid + FakeZbsCpuIsolationProvider provider + ZbsResourceAssignmentBackend backend + PrimaryStorageInventory first + PrimaryStorageInventory second + + @Override + void setup() { + useSpring(springSpec) + } + + @Override + void environment() { + env = HostEnv.oneHostEnv() + } + + @Override + void clean() { + ZbsResourceAssignmentGlobalConfig.PROVIDER_CALL_TIMEOUT.resetValue() + if (provider != null) { + provider.disable() + } + env.delete() + } + + @Override + void test() { + env.create { + host = env.inventoryByName("kvm") as HostInventory + provider = bean(FakeZbsCpuIsolationProvider.class) + backend = bean(ZbsResourceAssignmentBackend.class) + installKvmSimulators() + installZbsSerialSimulator() + provider.enable() + provider.setAutoApply(true) + + first = addZbs("zbs-assignment-1", "127.0.1.11") + serverUuid = physicalServerUuid(SERIAL) + provider.setFact(serverUuid, disabledFact()) + associateHost(SERIAL) + + verifyHyperconvergedDefaults() + verifyProviderUpdateNoopFailureAndTimeout() + verifySerialPriorityAndIpBackfill() + verifyCascadeKeepsSharedRelationAndReleasesLastRelation() + verifyForceDeleteForgetsLedgerAfterProviderFailure() + verifyAddonInfoMoveReleasesOldRelation() + verifyProviderAmbiguityIsRejected() + } + } + + private void installKvmSimulators() { + env.simulator(KVMConstant.KVM_HOST_NUMA_PATH) { + HostNUMANode node = new HostNUMANode() + node.nodeID = "0" + node.cpus = ["0", "1", "2", "3", "4", "5", "6", "7"] + node.onlineCpus = node.cpus + node.coreGroups = [ + ["0", "4"], ["1", "5"], + ["2", "6"], ["3", "7"] + ] + node.distance = ["10"] + node.free = SizeUnit.GIGABYTE.toByte(8) + node.size = SizeUnit.GIGABYTE.toByte(8) + KVMAgentCommands.GetHostNUMATopologyResponse response = + new KVMAgentCommands.GetHostNUMATopologyResponse() + response.topology = ["0": node] + return response + } + env.simulator(KvmPhysicalServerAdapter.APPLY_RESOURCE_CONTROL_PATH) { + HttpEntity entity -> + KvmPhysicalServerAdapter.ResourceControlAgentCommand command = + JSONObjectUtil.toObject( + entity.body, + KvmPhysicalServerAdapter.ResourceControlAgentCommand.class) + boolean release = command.operation == "RELEASE" + KvmPhysicalServerAdapter.ResourceControlAgentResponse response = + new KvmPhysicalServerAdapter.ResourceControlAgentResponse() + response.cpuSet = release ? "" : command.cpuSet + response.memory = command.memory == null + ? null : release ? 0L : command.memory + response.expectedServiceCount = 1 + response.coveredServiceCount = 1 + ResourceControlResult result = new ResourceControlResult() + result.state = release ? "DISABLED" : "READY" + result.cpuSet = response.cpuSet + result.memory = response.memory + response.results = [result] + return response + } + } + + private void installZbsSerialSimulator() { + env.simulator(ZbsStorageController.GET_FACTS_PATH) { + ZbsStorageController.GetFactsRsp response = + new ZbsStorageController.GetFactsRsp() + response.uuid = "zbs-resource-assignment-case" + response.version = "1.6.1-for-test" + response.success = true + return response + } + env.simulator(ZbsStorageController.GET_CAPACITY_PATH) { + HttpEntity entity -> + ZbsStorageController.GetCapacityCmd command = + JSONObjectUtil.toObject( + entity.body, + ZbsStorageController.GetCapacityCmd.class) + LogicalPoolInfo pool = new LogicalPoolInfo() + pool.logicalPoolName = command.logicalPoolNames[0] + pool.physicalPoolName = "pool1" + pool.capacity = SizeUnit.TERABYTE.toByte(1) + pool.allocatedSize = SizeUnit.GIGABYTE.toByte(1) + pool.usedSize = SizeUnit.GIGABYTE.toByte(1) + ZbsStorageController.GetCapacityRsp response = + new ZbsStorageController.GetCapacityRsp() + response.logicalPoolInfos = [pool] + return response + } + env.afterSimulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { + ZbsPrimaryStorageMdsBase.SyncMetadataRsp response, + HttpEntity entity -> + ZbsPrimaryStorageMdsBase.SyncMetadataCmd command = + JSONObjectUtil.toObject( + entity.body, + ZbsPrimaryStorageMdsBase.SyncMetadataCmd.class) + if (command.addr.startsWith("127.0.9.")) { + response.physicalServerSerialNumber = + "zbs-moved-physical-server-case" + } else if (command.addr != host.managementIp) { + response.physicalServerSerialNumber = SERIAL + } + return response + } + } + + private void verifyHyperconvergedDefaults() { + String targetUuid = serverUuid + refreshPhysicalServerResourceAssignments { + delegate.serverUuid = targetUuid + } + retryInSecs { + PhysicalServerResourceAssignmentInventory zbs = assignment( + serverUuid, "ZBS") + PhysicalServerResourceAssignmentInventory compute = assignment( + serverUuid, "COMPUTE") + assert zbs.state == "Synced" && + zbs.cpuSet == "2-3,6-7" : + "ZBS must own the rear 50 percent as complete SMT cores: " + + "expected=Synced/2-3,6-7 " + + "actual=${zbs.state}/${zbs.cpuSet}" + assert compute.state == "Synced" && + compute.cpuSet == "0-1,4-5" : + "COMPUTE shared CPUs must exclude the ZBS exclusive set: " + + "expected=Synced/0-1,4-5 " + + "actual=${compute.state}/${compute.cpuSet}" + } + assert provider.updateCalls >= 1 : + "default ZBS reconciliation must call the Provider update API: " + + "actualCalls=${provider.updateCalls}" + + def service = getPhysicalServerManagedServices { + delegate.serverUuid = targetUuid + }.services.find { it.roleType == "ZBS" } + assert service?.serviceName == "zbs-node" && + service.state == "RUNNING" : + "ZBS owner fact must be exposed as a managed service: " + + "actual=${service}" + assert !service.restartable : + "Cloud must not restart a Provider-managed ZBS service: " + + "actualRestartable=${service.restartable}" + + RefreshPhysicalServerResourceAssignmentsAction restart = + new RefreshPhysicalServerResourceAssignmentsAction( + sessionId: adminSession(), + serverUuid: serverUuid, + roleType: "ZBS", + serviceNames: ["zbs-node"]) + def result = restart.call() + assert result.error?.details?.contains( + "SERVICE_RESTART_NOT_SUPPORTED") : + "Provider-managed ZBS must reject Cloud restart: " + + "actual=${result.error}" + } + + private void verifyProviderUpdateNoopFailureAndTimeout() { + String targetUuid = serverUuid + updatePhysicalServerResourceAssignment { + delegate.serverUuid = targetUuid + roleType = "ZBS" + cpuSet = "3,7" + memory = SizeUnit.GIGABYTE.toByte(1) + } + retryInSecs { + PhysicalServerResourceAssignmentInventory current = assignment( + serverUuid, "ZBS") + assert current.state == "Synced" && + current.cpuSet == "3,7" && + current.memory == SizeUnit.GIGABYTE.toByte(1) : + "ZBS PATCH must be confirmed by a second Provider Query: " + + "actual=${current.state}/${current.cpuSet}/${current.memory}" + } + int updatesBeforeNoop = provider.updateCalls + refreshPhysicalServerResourceAssignments { + delegate.serverUuid = targetUuid + } + retryInSecs { + assert assignment(serverUuid, "ZBS").state == "Synced" : + "matching Provider fact must keep the Assignment Synced" + } + assert provider.updateCalls == updatesBeforeNoop : + "matching desired and observed facts must not write Provider state again: " + + "before=${updatesBeforeNoop} after=${provider.updateCalls}" + + provider.setAutoApply(false) + updatePhysicalServerResourceAssignment { + delegate.serverUuid = targetUuid + roleType = "ZBS" + cpuSet = "2,6" + } + retryInSecs { + assert assignment(serverUuid, "ZBS").state == "Unsynced" : + "Provider success without observed convergence must remain Unsynced" + } + provider.setAutoApply(true) + refreshPhysicalServerResourceAssignments { + delegate.serverUuid = targetUuid + } + retryInSecs { + PhysicalServerResourceAssignmentInventory recovered = assignment( + serverUuid, "ZBS") + assert recovered.state == "Synced" && + recovered.cpuSet == "2,6" : + "a later Provider convergence must recover the same ledger row: " + + "actual=${recovered.state}/${recovered.cpuSet}" + } + + verifyProviderTimeout(false) + verifyProviderTimeout(true) + } + + private void verifyProviderTimeout(boolean holdUpdate) { + ZbsResourceAssignmentGlobalConfig.PROVIDER_CALL_TIMEOUT.updateValue(1) + if (holdUpdate) { + provider.setFact(serverUuid, disabledFact()) + provider.holdNextUpdate() + } else { + provider.holdNextQuery() + } + ResourceControlCommand command = new ResourceControlCommand() + command.roleType = "ZBS" + command.operation = "APPLY" + command.cpuSet = "2,6" + command.memory = SizeUnit.GIGABYTE.toByte(1) + CountDownLatch completed = new CountDownLatch(1) + AtomicReference failure = new AtomicReference<>() + backend.apply( + serverUuid, + null, + command, + new ReturnValueCompletion(null) { + @Override + void success(ResourceControlResponse ignored) { + completed.countDown() + } + + @Override + void fail(ErrorCode errorCode) { + failure.set(errorCode) + completed.countDown() + } + }) + assert completed.await(5, TimeUnit.SECONDS) : + "Provider timeout must complete within the configured bound: " + + "phase=${holdUpdate ? 'Update' : 'Query'}" + assert failure.get()?.details?.contains("ZBS_PROVIDER_CALL_TIMEOUT") : + "Provider timeout must retain a typed Query/Update failure: " + + "phase=${holdUpdate ? 'Update' : 'Query'} " + + "actual=${failure.get()}" + if (holdUpdate) { + provider.completeHeldUpdate() + } else { + provider.completeHeldQuery() + } + provider.setAutoApply(true) + provider.setFact( + serverUuid, + readyFact("2,6", SizeUnit.GIGABYTE.toByte(1))) + ZbsResourceAssignmentGlobalConfig.PROVIDER_CALL_TIMEOUT.resetValue() + } + + private void verifySerialPriorityAndIpBackfill() { + ZbsNodeRefContributorImpl refs = bean( + ZbsNodeRefContributorImpl.class) + def serialRef = refs.bulkList([serverUuid])[serverUuid] + assert serialRef?.primaryStorageUuids?.contains(first.uuid) : + "a stable MDS serial must relate ZBS even when its address differs from Host IP: " + + "primaryStorageUuid=${first.uuid} actual=${serialRef?.primaryStorageUuids}" + + second = addZbs("zbs-assignment-ip-backfill", host.managementIp) + retryInSecs { + List infos = mdsInfos(second.uuid) + assert infos.size() == 1 && + infos[0].physicalServerSerialNumber == SERIAL : + "legacy address matching may only backfill the stable serial identity: " + + "expected=${SERIAL} actual=${infos*.physicalServerSerialNumber}" + } + def combined = refs.bulkList([serverUuid])[serverUuid] + assert combined.primaryStorageUuids.containsAll([ + first.uuid, second.uuid + ]) && combined.sourceRefCount == 2 : + "multiple ZBS relations on one machine must collapse into one node ref: " + + "actualPrimaryStorages=${combined.primaryStorageUuids} " + + "actualCount=${combined.sourceRefCount}" + } + + private void verifyCascadeKeepsSharedRelationAndReleasesLastRelation() { + int updatesBeforeFirstDelete = provider.updateCalls + DeletePrimaryStorageAction.Result firstDelete = + deletePrimaryStorage(first.uuid, "Permissive") + assert firstDelete.error == null : + "deleting one of multiple ZBS relations must succeed: " + + "actual=${firstDelete.error}" + assert provider.updateCalls == updatesBeforeFirstDelete : + "deleting a non-last relation must not disable machine isolation: " + + "before=${updatesBeforeFirstDelete} " + + "after=${provider.updateCalls}" + assert assignment(serverUuid, "ZBS") != null : + "the shared ZBS Assignment must survive while another relation exists" + + provider.disable() + DeletePrimaryStorageAction.Result blocked = + deletePrimaryStorage(second.uuid, "Permissive") + assert blocked.error?.details?.contains("ZBS_PROVIDER_UNAVAILABLE") : + "normal deletion of the last relation must stop if release cannot be verified: " + + "actual=${blocked.error}" + assert queryPrimaryStorage { + conditions = ["uuid=${second.uuid}"] + }.size() == 1 : + "blocked Cascade must preserve the last ZBS PrimaryStorage relation" + assert assignment(serverUuid, "ZBS").state == "Unsynced" : + "failed release must preserve an Unsynced ledger for retry" + + provider.enable() + provider.setAutoApply(true) + provider.setFact( + serverUuid, + readyFact("2,6", SizeUnit.GIGABYTE.toByte(1))) + DeletePrimaryStorageAction.Result deleted = + deletePrimaryStorage(second.uuid, "Permissive") + assert deleted.error == null : + "last relation deletion must continue after Provider release succeeds: " + + "actual=${deleted.error}" + retryInSecs { + assert assignments(serverUuid, "ZBS").isEmpty() : + "last relation Cascade must delete the ZBS Assignment ledger" + } + } + + private void verifyForceDeleteForgetsLedgerAfterProviderFailure() { + String targetUuid = serverUuid + PrimaryStorageInventory forced = addZbs( + "zbs-assignment-force-delete", "127.0.1.13") + provider.setFact(serverUuid, disabledFact()) + refreshPhysicalServerResourceAssignments { + delegate.serverUuid = targetUuid + } + retryInSecs { + assert assignment(serverUuid, "ZBS").state == "Synced" : + "force-delete scenario requires a converged ZBS Assignment first" + } + + provider.disable() + DeletePrimaryStorageAction.Result result = + deletePrimaryStorage(forced.uuid, "Enforcing") + assert result.error == null : + "Enforcing deletion must not be blocked by an unreachable Provider: " + + "actual=${result.error}" + retryInSecs { + assert assignments(serverUuid, "ZBS").isEmpty() : + "Enforcing last-relation deletion must not leave Role configuration" + } + provider.enable() + provider.setAutoApply(true) + } + + private void verifyAddonInfoMoveReleasesOldRelation() { + String targetUuid = serverUuid + provider.setFact(serverUuid, disabledFact()) + PrimaryStorageInventory moved = addZbs( + "zbs-assignment-addon-move", "127.0.1.14") + refreshPhysicalServerResourceAssignments { + delegate.serverUuid = targetUuid + } + retryInSecs { + assert assignment(serverUuid, "ZBS").state == "Synced" : + "addonInfo move scenario requires the old relation to be active" + } + + updateExternalPrimaryStorage { + uuid = moved.uuid + config = zbsConfig("127.0.9.14") + } + retryInSecs { + assert assignments(serverUuid, "ZBS").isEmpty() : + "moving the last MDS identity must release the old server Assignment before persist" + } + String movedServerUuid = physicalServerUuid( + "zbs-moved-physical-server-case") + assert movedServerUuid != serverUuid : + "a different stable serial must resolve to a different PhysicalServer: " + + "old=${serverUuid} new=${movedServerUuid}" + + DeletePrimaryStorageAction.Result cleanup = + deletePrimaryStorage(moved.uuid, "Enforcing") + assert cleanup.error == null : + "moved test PrimaryStorage must be force-cleanable: " + + "actual=${cleanup.error}" + retryInSecs { + assert assignments(movedServerUuid, "ZBS").isEmpty() : + "force cleanup must forget the moved server ledger" + } + } + + private void verifyProviderAmbiguityIsRejected() { + ZbsCpuIsolationProvider duplicate = [ + getProviderType: { "DUPLICATE" }, + isAvailable: { ref -> true }, + query: { ref, completion -> + completion.success(disabledFact()) + }, + update: { ref, update, completion -> + completion.success() + } + ] as ZbsCpuIsolationProvider + bean(PluginRegistry.class).defineDynamicExtension( + ZbsCpuIsolationProvider.class, duplicate) + + PrimaryStorageInventory ambiguous = addZbs( + "zbs-assignment-provider-ambiguous", "127.0.1.15") + retryInSecs { + backend.refreshAssociations([serverUuid]) + assert backend.getState(serverUuid).name() == "UNAVAILABLE" : + "two available Providers must disable execution for the node" + assert backend.getUnavailableReason(serverUuid) == + "ZBS_PROVIDER_AMBIGUOUS" : + "Provider ambiguity must expose a deterministic reason: " + + "actual=${backend.getUnavailableReason(serverUuid)}" + } + deletePrimaryStorage(ambiguous.uuid, "Enforcing") + } + + private PrimaryStorageInventory addZbs(String name, String address) { + return addExternalPrimaryStorage { + delegate.zoneUuid = host.zoneUuid + delegate.name = name + delegate.identity = "zbs" + delegate.defaultOutputProtocol = "CBD" + delegate.config = zbsConfig(address) + delegate.url = "zbs" + } as PrimaryStorageInventory + } + + private String zbsConfig(String address) { + return JSONObjectUtil.toJsonString([ + mdsUrls: ["root:password@${address}".toString()], + logicalPoolName: "lpool1" + ]) + } + + private void associateHost(String serialNumber) { + List tags = querySystemTag { + conditions = [ + "resourceUuid=${host.uuid}", + "tag~=systemSerialNumber::%" + ] + } as List + if (tags.isEmpty()) { + createSystemTag { + resourceUuid = host.uuid + resourceType = "HostVO" + tag = "systemSerialNumber::${serialNumber}" + } + } else { + updateSystemTag { + uuid = tags[0].uuid + tag = "systemSerialNumber::${serialNumber}" + } + } + bean(KvmPhysicalServerAdapter.class).associate( + org.zstack.header.host.HostInventory.valueOf( + dbFindByUuid(host.uuid, HostVO.class))) + retryInSecs { + HostInventory current = (queryHost { + conditions = ["uuid=${host.uuid}"] + } as List)[0] + assert current.serverUuid == serverUuid : + "Host and ZBS serials must compose onto one PhysicalServer: " + + "expected=${serverUuid} actual=${current.serverUuid}" + host = current + } + } + + private String physicalServerUuid(String serialNumber) { + AtomicReference found = new AtomicReference<>() + retryInSecs { + def servers = queryPhysicalServer { + conditions = ["serialNumber=${serialNumber}"] + } + assert servers.size() == 1 : + "one stable serial must resolve to exactly one PhysicalServer: " + + "serial=${serialNumber} actual=${servers.size()}" + found.set(servers[0].uuid) + } + return found.get() + } + + private List mdsInfos(String primaryStorageUuid) { + String addonInfo = Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, primaryStorageUuid) + .findValue() + assert addonInfo != null : + "the ZBS PrimaryStorage must exist while reading addonInfo: " + + "uuid=${primaryStorageUuid} actual=missing" + return JSONObjectUtil.toObject( + addonInfo, AddonInfo.class).mdsInfos + } + + private DeletePrimaryStorageAction.Result deletePrimaryStorage( + String uuid, String mode) { + return new DeletePrimaryStorageAction( + sessionId: adminSession(), + uuid: uuid, + deleteMode: mode).call() + } + + private PhysicalServerResourceAssignmentInventory assignment( + String targetServerUuid, String roleType) { + List rows = assignments( + targetServerUuid, roleType) + assert rows.size() == 1 : + "server and Role must identify exactly one Assignment: " + + "serverUuid=${targetServerUuid} roleType=${roleType} " + + "actual=${rows.size()}" + return rows[0] + } + + private List assignments( + String targetServerUuid, String roleType) { + return queryPhysicalServerResourceAssignment { + conditions = [ + "serverUuid=${targetServerUuid}", + "roleType=${roleType}" + ] + } as List + } + + private static ZbsCpuIsolationFact disabledFact() { + ZbsCpuIsolationFact fact = new ZbsCpuIsolationFact() + fact.isolationState = "DISABLED" + fact.configuredCpuSet = "" + fact.effectiveCpuSet = "" + fact.expectedServiceCount = 1 + fact.coveredServiceCount = 1 + fact.serviceReady = true + return fact + } + + private static ZbsCpuIsolationFact readyFact( + String cpuSet, Long memory) { + ZbsCpuIsolationFact fact = new ZbsCpuIsolationFact() + fact.isolationState = "READY" + fact.configuredCpuSet = cpuSet + fact.effectiveCpuSet = cpuSet + fact.memory = memory + fact.expectedServiceCount = 1 + fact.coveredServiceCount = 1 + fact.serviceReady = true + return fact + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 589e47918a0..68c9d2aaeca 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -206,9 +206,43 @@ class ZbsPrimaryStorageCase extends SubCase { testMdsReconnectAfterMaximumPingFailures() testGetBackingChainNormalizesCbdParentUri() testBatchStatsNormalizesCbdInstallPath() + testMdsPhysicalServerSerialPersistsAcrossReconnect() } } + void testMdsPhysicalServerSerialPersistsAcrossReconnect() { + env.afterSimulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { + ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp, HttpEntity e -> + def cmd = JSONObjectUtil.toObject( + e.body, ZbsPrimaryStorageMdsBase.SyncMetadataCmd.class) + rsp.physicalServerSerialNumber = "PS-SN-${cmd.addr}" + return rsp + } + + reconnectPrimaryStorage { + uuid = ps.uuid + } + assert mdsInfos().every { + it.physicalServerSerialNumber == "ps-sn-${it.addr}" + } + + env.cleanAfterSimulatorHandlers() + reconnectPrimaryStorage { + uuid = ps.uuid + } + assert mdsInfos().every { + it.physicalServerSerialNumber == "ps-sn-${it.addr}" + } + } + + private List mdsInfos() { + String serialized = Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, ps.uuid) + .findValue() + return JSONObjectUtil.toObject(serialized, AddonInfo.class).mdsInfos + } + void testSyncPrimaryStorageCapacityConcurrently() { def threads = new ArrayList<>() def success_cnt = new AtomicInteger(0) diff --git a/test/src/test/groovy/org/zstack/test/unittest/physicalserver/PhysicalServerResourceModelCase.groovy b/test/src/test/groovy/org/zstack/test/unittest/physicalserver/PhysicalServerResourceModelCase.groovy new file mode 100644 index 00000000000..aa8db75bf4f --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/unittest/physicalserver/PhysicalServerResourceModelCase.groovy @@ -0,0 +1,384 @@ +package org.zstack.test.unittest.physicalserver + +import org.junit.Test +import org.zstack.core.Platform +import org.zstack.core.componentloader.PluginRegistry +import org.zstack.header.physicalserver.PhysicalServerCpuSet +import org.zstack.header.physicalserver.PhysicalServerCpuTopology +import org.zstack.header.physicalserver.PhysicalServerNumaNode +import org.zstack.header.physicalserver.PhysicalServerResourceApplicationMode +import org.zstack.header.physicalserver.PhysicalServerResourceControlAdapter +import org.zstack.header.physicalserver.PhysicalServerResourceIsolationMode +import org.zstack.header.physicalserver.ResourceConsumerHandle +import org.zstack.header.physicalserver.RoleServiceManifest +import org.zstack.physicalserver.PhysicalServerCpuPlanner +import org.zstack.physicalserver.PhysicalServerReasonCodes +import org.zstack.physicalserver.PhysicalServerResourceAssignmentReconciler +import org.zstack.physicalserver.PhysicalServerResourceControlAdapterRegistry +import org.zstack.physicalserver.PhysicalServerResourceControlReconciler +import org.zstack.storage.zbs.ZbsResourceAssignmentBackend +import org.zstack.storage.zbs.AddonInfo +import org.zstack.storage.zbs.ZbsNodeRefContributorImpl + +import static org.mockito.Mockito.mock +import static org.mockito.Mockito.when + +class PhysicalServerResourceModelCase { + @Test + void testMachineSerialNormalizationRejectsFirmwarePlaceholders() { + assert Platform.normalizeMachineSerialNumber(" PS-SN-01 ") == + "ps-sn-01" : + "machine identity must be stable across case and whitespace: " + + "expected=ps-sn-01 actual=${Platform.normalizeMachineSerialNumber(' PS-SN-01 ')}" + [null, "", "N/A", "unknown", "To Be Filled", "default string"].each { + assert Platform.normalizeMachineSerialNumber(it) == null : + "firmware placeholder must not create a PhysicalServer: " + + "serial=${it} actual=${Platform.normalizeMachineSerialNumber(it)}" + } + } + + @Test + void testCpuSetNormalizesWithoutExpandingUntrustedRanges() { + assert PhysicalServerCpuSet.normalize(" 3,1-2,2,5-6 ") == "1-3,5-6" : + "CPUSet must merge and sort overlapping ranges: " + + "expected=1-3,5-6 actual=${PhysicalServerCpuSet.normalize(' 3,1-2,2,5-6 ')}" + assert PhysicalServerCpuSet.count("1-3,5-6") == 5 : + "CPUSet count must use normalized ranges: expected=5 " + + "actual=${PhysicalServerCpuSet.count('1-3,5-6')}" + assert PhysicalServerCpuSet.union("0-1", "1-3") == "0-3" : + "CPUSet union must merge adjacent ranges: expected=0-3 " + + "actual=${PhysicalServerCpuSet.union('0-1', '1-3')}" + + assertFailure("CPU_SET_INVALID") { + PhysicalServerCpuSet.normalize("1-") + } + assertFailure("CPU_SET_INVALID") { + PhysicalServerCpuSet.parse("0-2147483647") + } + assertFailure("outside the online topology") { + PhysicalServerCpuSet.parse("0-4", [0, 1, 2, 3] as Set) + } + } + + @Test + void testTopologyRequiresEveryOnlineCpuInExactlyOneCoreGroup() { + PhysicalServerCpuTopology topology = topology() + + assert topology.onlineCpus == ([0, 1, 2, 3, 4, 5, 6, 7] as Set) : + "topology must retain every online CPU exactly once: " + + "expected=0-7 actual=${topology.onlineCpus}" + assert topology.fingerprint() == "0:0,4;0:1,5;1:2,6;1:3,7" : + "topology fingerprint must be stable across NUMA map order: " + + "expected=0:0,4;0:1,5;1:2,6;1:3,7 actual=${topology.fingerprint()}" + + PhysicalServerNumaNode missingCpu = node( + "0", ["0", "1"], [["0"]]) + assertFailure("do not cover every online CPU") { + PhysicalServerCpuTopology.from(["0": missingCpu]) + } + + PhysicalServerNumaNode duplicatedCpu = node( + "0", ["0", "1"], [["0"], ["0", "1"]]) + assertFailure("appears in multiple core groups") { + PhysicalServerCpuTopology.from(["0": duplicatedCpu]) + } + } + + @Test + void testExclusiveCpuPlanningPreservesWholeCoresAndCpuZero() { + PhysicalServerCpuPlanner planner = new PhysicalServerCpuPlanner() + PhysicalServerCpuTopology topology = topology() + + String shared = planner.validateAndNormalize( + PhysicalServerResourceIsolationMode.SHARED, + "0-1,4-5", topology, [2, 3, 6, 7]) + assert shared == "0-1,4-5" : + "shared CPUs outside exclusive reservations must remain valid: " + + "expected=0-1,4-5 actual=${shared}" + + String exclusive = planner.validateAndNormalize( + PhysicalServerResourceIsolationMode.EXCLUSIVE, + "2-3,6-7", topology, Collections.emptySet()) + assert exclusive == "2-3,6-7" : + "exclusive CPUs must accept complete SMT core groups: " + + "expected=2-3,6-7 actual=${exclusive}" + + assertFailure("CPU_SET_CONFLICT") { + planner.validateAndNormalize( + PhysicalServerResourceIsolationMode.SHARED, + "0-2", topology, [2, 6]) + } + assertFailure("CPU_SIBLING_SPLIT") { + planner.validateAndNormalize( + PhysicalServerResourceIsolationMode.EXCLUSIVE, + "2", topology, Collections.emptySet()) + } + assertFailure("CPU_ZERO_RESERVED") { + planner.validateAndNormalize( + PhysicalServerResourceIsolationMode.EXCLUSIVE, + "0,4", topology, Collections.emptySet()) + } + } + + @Test + void testZbsDefaultUsesRearHalfOfCompleteCoreGroups() { + ZbsResourceAssignmentBackend zbs = + new ZbsResourceAssignmentBackend() + + String selected = zbs.getDefaultCpuSet( + topology(), Collections.emptySet()) + assert selected == "2-3,6-7" : + "ZBS default must reserve the rear half by complete SMT cores: " + + "expected=2-3,6-7 actual=${selected}" + + PhysicalServerCpuTopology oneCore = PhysicalServerCpuTopology.from([ + "0": node("0", ["0", "4"], [["0", "4"]]) + ]) + assertFailure("CPU_TOPOLOGY_TOO_SMALL") { + zbs.getDefaultCpuSet(oneCore, Collections.emptySet()) + } + + assertFailure("ZBS_NODE_RELATION_SOURCE_INVALID") { + new ZbsNodeRefContributorImpl().resolveServerUuids( + "invalid-primary-storage", new AddonInfo()) + } + } + + @Test + void testRoleManifestsDefineStableHandlesWithoutAdapterCodeChanges() { + RoleServiceManifest compute = RoleServiceManifest.load( + "physical-server-roles/compute.yaml", + "COMPUTE", + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES) + List computeCore = compute.handles( + "host-agent:core", "host-agent:aux", false) + List computeAll = compute.handles( + "host-agent:core", "host-agent:aux", true) + + assert compute.sliceName == "zstack-compute.slice" : + "compute manifest must preserve its stable Role slice: " + + "expected=zstack-compute.slice actual=${compute.sliceName}" + assert computeCore*.serviceName == + ["kvmagent", "virtlogd", "network-agent", "sharedblock-agent"] : + "compute core handles must exclude auxiliary monitoring services: " + + "actual=${computeCore*.serviceName}" + assert computeAll*.serviceName.contains("node-exporter") : + "compute manifest must add auxiliary services by configuration: " + + "actual=${computeAll*.serviceName}" + assert computeAll.find { it.serviceName == "node-exporter" }.restartable : + "manifest restartability must reach the generated handle: " + + "service=node-exporter actual=${computeAll.find { it.serviceName == 'node-exporter' }.restartable}" + + List selected = compute.handlesByServiceNames( + ["node-exporter"], "host-agent:core", "host-agent:aux", true, + Collections.emptyMap()) + assert selected*.serviceName == ["node-exporter"] : + "targeted restart must resolve only explicitly selected services: " + + "expected=[node-exporter] actual=${selected*.serviceName}" + assertFailure("are not defined by role") { + compute.handlesByServiceNames( + ["unknown-service"], "host-agent:core", "host-agent:aux", + true, Collections.emptyMap()) + } + + RoleServiceManifest owner = RoleServiceManifest.load( + "physical-server-roles/test-owner.yaml", + "TEST_OWNER", + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES) + ResourceConsumerHandle ownerHandle = owner.handles( + "owner:core", "owner:aux", true, + [ownerPidFile: "/run/test-owner.pid"])[0] + assert ownerHandle.handleType == ResourceConsumerHandle.OWNER_PID_FILE : + "valueFrom manifest must retain OWNER_PID_FILE identity: " + + "expected=OWNER_PID_FILE actual=${ownerHandle.handleType}" + assert ownerHandle.value == "/run/test-owner.pid" : + "valueFrom must resolve from adapter supplied values: " + + "expected=/run/test-owner.pid actual=${ownerHandle.value}" + assert ownerHandle.expectedCommandToken == "/usr/bin/test-owner" : + "PID-file handles must carry the trusted command token: " + + "expected=/usr/bin/test-owner actual=${ownerHandle.expectedCommandToken}" + assertFailure("cannot be resolved") { + owner.handles( + "owner:core", "owner:aux", true, Collections.emptyMap()) + } + + RoleServiceManifest zbs = RoleServiceManifest.load( + "physical-server-roles/zbs.yaml", + "ZBS", + PhysicalServerResourceApplicationMode.PROVIDER_MANAGED) + assertFailure("does not use resource handles") { + zbs.handles("zbs", "zbs", true) + } + assertFailure("does not match expected roleType") { + RoleServiceManifest.load( + "physical-server-roles/compute.yaml", + "IMAGE_STORE", + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES) + } + assertFailure("provider-managed role cannot define sliceName") { + RoleServiceManifest.load( + "physical-server-roles/invalid-provider-slice.yaml", + "INVALID_PROVIDER", + PhysicalServerResourceApplicationMode.PROVIDER_MANAGED) + } + assertFailure("restartable requires a systemd handle") { + RoleServiceManifest.load( + "physical-server-roles/invalid-owner-restart.yaml", + "INVALID_OWNER", + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES) + } + } + + @Test + void testAdapterRegistryRejectsAmbiguousAndBrokenTopologyRoles() { + PhysicalServerResourceControlAdapter compute = adapter("COMPUTE") + PhysicalServerResourceControlAdapter imageStore = adapter( + "IMAGE_STORE", "COMPUTE") + PluginRegistry pluginRegistry = registry([compute, imageStore]) + + PhysicalServerResourceControlAdapterRegistry loaded = + PhysicalServerResourceControlAdapterRegistry.load(pluginRegistry) + assert loaded.orderedRoleTypes() == ["COMPUTE", "IMAGE_STORE"] : + "valid adapters must be ordered without enum registration: " + + "expected=[COMPUTE, IMAGE_STORE] actual=${loaded.orderedRoleTypes()}" + + loaded = PhysicalServerResourceControlAdapterRegistry.load(registry([ + compute, + adapter("IMAGE_STORE", "COMPUTE"), + adapter("IMAGE_STORE", "COMPUTE") + ])) + assert !loaded.contains("IMAGE_STORE") : + "duplicate Role adapters must not enter the executable registry: " + + "roleType=IMAGE_STORE actual=${loaded.orderedRoleTypes()}" + assert loaded.getError("IMAGE_STORE").contains( + "RESOURCE_ASSIGNMENT_ADAPTER_AMBIGUOUS") : + "duplicate Role adapters must expose a deterministic reason: " + + "actual=${loaded.getError('IMAGE_STORE')}" + + loaded = PhysicalServerResourceControlAdapterRegistry.load(registry([ + adapter("STORAGE", "MISSING") + ])) + assert !loaded.contains("STORAGE") : + "adapter with a missing topology provider must be disabled: " + + "actual=${loaded.orderedRoleTypes()}" + assert loaded.getError("STORAGE").contains( + "RESOURCE_ASSIGNMENT_TOPOLOGY_ROLE_UNAVAILABLE") : + "missing topology dependency must expose its reason: " + + "actual=${loaded.getError('STORAGE')}" + + PhysicalServerResourceControlReconciler reconciler = + new PhysicalServerResourceControlReconciler() + def unavailableReason = reconciler.class.getDeclaredMethod( + "adapterUnavailableReason", String, + PhysicalServerResourceControlAdapterRegistry) + unavailableReason.accessible = true + assert unavailableReason.invoke( + reconciler, "UNKNOWN", loaded) == + "RESOURCE_ASSIGNMENT_ADAPTER_MISSING" : + "a Role absent from the registry must retain a stable missing-adapter reason" + + PhysicalServerResourceControlAdapterRegistry ambiguous = + PhysicalServerResourceControlAdapterRegistry.load(registry([ + adapter("IMAGE_STORE", "COMPUTE"), + adapter("IMAGE_STORE", "COMPUTE"), + compute + ])) + assert unavailableReason.invoke( + reconciler, "IMAGE_STORE", ambiguous) == + "RESOURCE_ASSIGNMENT_ADAPTER_AMBIGUOUS" : + "a broken Role must retain the registry's typed reason without diagnostic suffixes" + } + + @Test + void testRefreshFailureBackoffIsBoundedAndKeepsRetrying() { + PhysicalServerResourceAssignmentReconciler reconciler = + new PhysicalServerResourceAssignmentReconciler() + def nextDelay = reconciler.class.getDeclaredMethod( + "nextRefreshRetryDelay") + nextDelay.accessible = true + + (1..10).each { int attempt -> + int cappedAttempt = Math.min(attempt, 8) + long lower = 100L << (cappedAttempt - 1) + long upper = Math.min(30000L, lower * 2) + long actual = nextDelay.invoke(reconciler) as long + assert actual >= lower && actual <= upper : + "refresh retry must use bounded exponential jitter: " + + "attempt=${attempt} expected=${lower}..${upper} actual=${actual}" + } + } + + @Test + void testReasonCodeUsesTheDeepestTypedFailure() { + String details = "wrapper: RESOURCE_CONTROL_APPLY_FAILED: " + + "CPUSET_CONTROLLER_NOT_DELEGATED: target missing" + String actual = PhysicalServerReasonCodes.from(details) + + assert actual == "CPUSET_CONTROLLER_NOT_DELEGATED" : + "the persisted reason must identify the deepest actionable failure: " + + "expected=CPUSET_CONTROLLER_NOT_DELEGATED actual=${actual}" + assert PhysicalServerReasonCodes.from((String) null) == + "RESOURCE_ASSIGNMENT_INVALID" : + "untyped failures must use the stable fallback reason: " + + "expected=RESOURCE_ASSIGNMENT_INVALID " + + "actual=${PhysicalServerReasonCodes.from((String) null)}" + } + + private static PhysicalServerCpuTopology topology() { + return PhysicalServerCpuTopology.from([ + "1": node("1", ["2", "3", "6", "7"], + [["2", "6"], ["3", "7"]]), + "0": node("0", ["0", "1", "4", "5"], + [["0", "4"], ["1", "5"]]) + ]) + } + + private static PhysicalServerNumaNode node( + String nodeId, List online, + List> coreGroups) { + PhysicalServerNumaNode node = new PhysicalServerNumaNode() + node.nodeId = nodeId + node.onlineCpus = online + node.coreGroups = coreGroups + return node + } + + private static PhysicalServerResourceControlAdapter adapter( + String roleType, String topologyRoleType = null) { + String topologyRole = topologyRoleType ?: roleType + PhysicalServerResourceControlAdapter adapter = mock( + PhysicalServerResourceControlAdapter.class) + when(adapter.getRoleType()).thenReturn(roleType) + when(adapter.getIsolationMode()).thenReturn( + PhysicalServerResourceIsolationMode.SHARED) + when(adapter.getApplicationMode()).thenReturn( + PhysicalServerResourceApplicationMode.RESOURCE_HANDLES) + when(adapter.getTopologyRoleType()).thenReturn(topologyRole) + return adapter + } + + private static PluginRegistry registry( + List adapters) { + PluginRegistry registry = mock(PluginRegistry.class) + when(registry.getExtensionList( + PhysicalServerResourceControlAdapter.class)).thenReturn(adapters) + return registry + } + + private static void assertFailure( + String expectedMessage, Closure operation) { + Throwable failure = null + try { + operation.call() + } catch (Throwable error) { + failure = error + } + assert failure != null : + "operation must fail with a typed validation error: " + + "expectedMessage=${expectedMessage} actual=no failure" + assert failure.message?.contains(expectedMessage) : + "validation failure must expose the expected reason: " + + "expectedMessage=${expectedMessage} " + + "actualType=${failure.class.name} actualMessage=${failure.message}" + } +} diff --git a/test/src/test/resources/physical-server-roles/invalid-owner-restart.yaml b/test/src/test/resources/physical-server-roles/invalid-owner-restart.yaml new file mode 100644 index 00000000000..75d58d3cd7a --- /dev/null +++ b/test/src/test/resources/physical-server-roles/invalid-owner-restart.yaml @@ -0,0 +1,10 @@ +roleType: INVALID_OWNER +applicationMode: RESOURCE_HANDLES +sliceName: invalid-owner.slice +services: + - name: invalid-owner + handleType: OWNER_PID_FILE + value: /run/invalid-owner.pid + required: true + restartable: true + expectedCommandToken: /usr/bin/invalid-owner diff --git a/test/src/test/resources/physical-server-roles/invalid-provider-slice.yaml b/test/src/test/resources/physical-server-roles/invalid-provider-slice.yaml new file mode 100644 index 00000000000..1943ad0276b --- /dev/null +++ b/test/src/test/resources/physical-server-roles/invalid-provider-slice.yaml @@ -0,0 +1,5 @@ +roleType: INVALID_PROVIDER +applicationMode: PROVIDER_MANAGED +sliceName: invalid-provider.slice +services: + - name: invalid-provider diff --git a/test/src/test/resources/physical-server-roles/test-owner.yaml b/test/src/test/resources/physical-server-roles/test-owner.yaml new file mode 100644 index 00000000000..da660a58aee --- /dev/null +++ b/test/src/test/resources/physical-server-roles/test-owner.yaml @@ -0,0 +1,10 @@ +roleType: TEST_OWNER +applicationMode: RESOURCE_HANDLES +sliceName: zstack-test-owner.slice +services: + - name: test-owner + handleType: OWNER_PID_FILE + valueFrom: ownerPidFile + required: true + restartable: false + expectedCommandToken: /usr/bin/test-owner diff --git a/test/src/test/resources/springConfigXml/Kvm.xml b/test/src/test/resources/springConfigXml/Kvm.xml index ec757841d12..fdf79430102 100755 --- a/test/src/test/resources/springConfigXml/Kvm.xml +++ b/test/src/test/resources/springConfigXml/Kvm.xml @@ -109,6 +109,24 @@ + + + + + + + + + + + + + + + + + diff --git a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy index 74c3ac573dc..be9ce1be410 100644 --- a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy +++ b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy @@ -22,7 +22,6 @@ abstract class ApiHelper { c.delegate = a c() - if (System.getProperty("apipath") != null) { if (a.apiId == null) { a.apiId = Platform.uuid @@ -34608,6 +34607,88 @@ abstract class ApiHelper { } + def queryPhysicalServer(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryPhysicalServerAction.class) Closure c) { + def a = new org.zstack.sdk.QueryPhysicalServerAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def getPhysicalServerManagedServices(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.GetPhysicalServerManagedServicesAction.class) Closure c) { + def a = new org.zstack.sdk.GetPhysicalServerManagedServicesAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + return out + } else { + return errorOut(a.call()) + } + } + + + def queryPhysicalServerResourceAssignment(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryPhysicalServerResourceAssignmentAction.class) Closure c) { + def a = new org.zstack.sdk.QueryPhysicalServerResourceAssignmentAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def queryPhysicalSwitch(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryPhysicalSwitchAction.class) Closure c) { def a = new org.zstack.sdk.QueryPhysicalSwitchAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -38799,6 +38880,33 @@ abstract class ApiHelper { } + def refreshPhysicalServerResourceAssignments(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsAction.class) Closure c) { + def a = new org.zstack.sdk.RefreshPhysicalServerResourceAssignmentsAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def refreshPluginDrivers(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.RefreshPluginDriversAction.class) Closure c) { def a = new org.zstack.sdk.RefreshPluginDriversAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -47169,6 +47277,32 @@ abstract class ApiHelper { } + def updatePhysicalServerResourceAssignment(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.UpdatePhysicalServerResourceAssignmentAction.class) Closure c) { + def a = new org.zstack.sdk.UpdatePhysicalServerResourceAssignmentAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def updatePluginSecretResourcePool(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.UpdatePluginSecretResourcePoolAction.class) Closure c) { def a = new org.zstack.sdk.UpdatePluginSecretResourcePoolAction() a.sessionId = Test.currentEnvSpec?.session?.uuid diff --git a/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy b/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy index a8e6812c992..7ef93c6871c 100755 --- a/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy @@ -114,6 +114,10 @@ class SpringSpec { include("cbd.xml") } + void physicalServer() { + include("physicalServer.xml") + } + void smp() { include("sharedMountPointPrimaryStorage.xml") }