From db4612ca3f32171b0bbed865d2698ba56a4faab6 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Mon, 4 May 2026 15:42:13 +0200 Subject: [PATCH 01/14] linstor: support shared (thick LVM) storage pools Resources in a LINSTOR shared storage pool live on one shared LUN and are active on at most one node; there is no DRBD replication or dual-primary. Handle them in the KVM storage adaptor and MS driver: - resolve the actual resource-definition name (template resources can be shared under a different name) before connect/copy operations - on live migration create and activate the resource on the destination node instead of setting DRBD allow-two-primaries - on disconnect delete the local resource only if another active resource exists (during live migration two resources are active at the same time) - getDevicePath: only accept resources that expose a device path, inactive shared resources have none - createResourceBase returns the actually used resource name --- .../kvm/storage/LinstorStorageAdaptor.java | 133 +++++++++++++----- .../LinstorPrimaryDataStoreDriverImpl.java | 13 +- .../storage/datastore/util/LinstorUtil.java | 12 +- 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 848bdc60d664..421adb7d6898 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -51,6 +51,7 @@ import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; import com.linbit.linstor.api.model.ResourceConnectionModify; +import com.linbit.linstor.api.model.ResourceCreate; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroupSpawn; @@ -239,21 +240,22 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Qemu String foundRscName = resourceDefinition != null ? resourceDefinition.getName() : rscName; + makeResourceAvailable(api, foundRscName, false); + // query linstor for the device path List resources = api.viewResources( - Collections.emptyList(), + Collections.singletonList(localNodeName), Collections.singletonList(foundRscName), Collections.emptyList(), null, null, null); - makeResourceAvailable(api, foundRscName, false); - if (!resources.isEmpty() && !resources.get(0).getVolumes().isEmpty()) { final String devPath = LinstorUtil.getDevicePathFromResource(resources.get(0)); logger.info("Linstor: Created drbd device: " + devPath); - final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, name, pool); + final String diskName = foundRscName.substring(LinstorUtil.RSC_PREFIX.length()); + final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, diskName, pool); kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); long allocatedKib = resources.get(0).getVolumes().get(0).getAllocatedSizeKib() != null ? resources.get(0).getVolumes().get(0).getAllocatedSizeKib() : 0; @@ -321,6 +323,37 @@ private void allow2PrimariesIfInUse(DevelopersApi api, String rscName) throws Ap } } + private boolean usesDrbd(DevelopersApi api, String rscName) throws ApiException { + List rscs = api.viewResources( + Collections.emptyList(), + Collections.singletonList(rscName), + null, + null, + null, + null); + + for (ResourceWithVolumes rsc : rscs) { + if (rsc.getLayerObject().getDrbd() != null) { + return true; + } + } + return false; + } + + + private String findCorrectTemplateName(DevelopersApi api, String rscName, KVMStoragePool pool ) { + String templateName = rscName; + LinstorStoragePool lpool = (LinstorStoragePool) pool; + try { + ResourceDefinition resourceDefinition = LinstorUtil.findResourceDefinition( + api, rscName, lpool.getResourceGroup()); + templateName = resourceDefinition != null ? resourceDefinition.getName() : rscName; + } catch (ApiException e) { + logger.error("Error finding resource definition for {}", rscName); + } + return templateName; + } + @Override public boolean connectPhysicalDisk( String volumePath, KVMStoragePool pool, Map details, boolean isVMMigration) @@ -333,20 +366,36 @@ public boolean connectPhysicalDisk( final DevelopersApi api = getLinstorAPI(pool); String rscName; + boolean usesDrbd; try { - rscName = getLinstorRscName(volumePath); - - ResourceMakeAvailable rma = new ResourceMakeAvailable(); - ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); - checkLinstorAnswersThrow(answers); + rscName = findCorrectTemplateName(api, getLinstorRscName(volumePath), pool); + usesDrbd = usesDrbd(api, rscName); + + if (isVMMigration && !usesDrbd) { + // create the resource on the new node, noop if already exists + ResourceCreate rc = new ResourceCreate(); + Resource rsc = new Resource(); + rsc.setNodeName(localNodeName); + rc.setResource(rsc); + ApiCallRcList answers = api.resourceCreateOnNode(rscName, localNodeName, rc); + checkLinstorAnswersThrow(answers); + // activate resource if there already was an inactive resource here + answers = api.activateRsc(rscName, localNodeName); + checkLinstorAnswersThrow(answers); + } else { + // DRBD resources just work with a diskless and don't have the activate/deactivate concept + ResourceMakeAvailable rma = new ResourceMakeAvailable(); + ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); + checkLinstorAnswersThrow(answers); + } } catch (ApiException apiEx) { logger.error(apiEx); throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } - if (isVMMigration) { + if (isVMMigration && usesDrbd) { try { allow2PrimariesIfInUse(api, rscName); } catch (ApiException apiEx) { @@ -393,13 +442,26 @@ private void removeTwoPrimariesProps(DevelopersApi api, String inUseNode, String removeTwoPrimariesRcProps(api, rscName, inUseNode, deleteProps); } + private void deleteUnusedSharedResources(DevelopersApi api, Resource nodeRsc) { + try { + List rscs = api.resourceList(nodeRsc.getName(), null, null); + if (rscs != null && rscs.stream() + .filter(rsc -> rsc.getFlags() == null || !rsc.getFlags().contains(ApiConsts.FLAG_RSC_INACTIVE)) + .count() > 1) { + api.resourceDelete(nodeRsc.getName(), localNodeName, true); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + } + } + private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) { if (volumePath == null) { return false; } - logger.debug("Linstor: Using storage pool: " + pool.getUuid()); + logger.debug("Linstor: Using storage pool: {}", pool.getUuid()); final DevelopersApi api = getLinstorAPI(pool); Optional optRsc; @@ -423,33 +485,40 @@ private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) if (optRsc.isPresent()) { Resource rsc = optRsc.get(); - try { - String inUseNode = LinstorUtil.isResourceInUse(api, rsc.getName()); - if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { - removeTwoPrimariesProps(api, inUseNode, rsc.getName()); + + // if it is a non DRBD resource(shared) we need to check if we are the last active one + // on live migrate, we can have 2 active resources at the same time + if (rsc.getLayerObject().getDrbd() == null) { + deleteUnusedSharedResources(api, rsc); + } else { + try { + String inUseNode = LinstorUtil.isResourceInUse(api, rsc.getName()); + if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { + removeTwoPrimariesProps(api, inUseNode, rsc.getName()); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal } - } catch (ApiException apiEx) { - logger.error(apiEx.getBestMessage()); - // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal - } - try { - // if diskless resource remove it, in the worst case it will be transformed to a tiebreaker - if (rsc.getFlags() != null && - rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) && - !rsc.getFlags().contains(ApiConsts.FLAG_TIE_BREAKER)) { - ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName, true); - logLinstorAnswers(delAnswers); + try { + // if diskless resource remove it, in the worst case it will be transformed to a tiebreaker + if (rsc.getFlags() != null && + rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) && + !rsc.getFlags().contains(ApiConsts.FLAG_TIE_BREAKER)) { + ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName, true); + logLinstorAnswers(delAnswers); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal } - } catch (ApiException apiEx) { - logger.error(apiEx.getBestMessage()); - // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal } return true; } - logger.warn("Linstor: Couldn't find resource for this path: " + volumePath); + logger.warn("Linstor: Couldn't find resource for this path: {}", volumePath); return false; } @@ -654,14 +723,14 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null); final DevelopersApi api = getLinstorAPI(destPools); - setRscDfnAuxProperties(api, disk, destPools, name); + setRscDfnAuxProperties(api, disk, destPools, dstDisk.getName()); logger.debug("Linstor.copyPhysicalDisk: dstPath: {}", dstDisk.getPath()); final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath()); destFile.setFormat(dstDisk.getFormat()); destFile.setSize(disk.getVirtualSize()); - boolean zeroedDevice = LinstorUtil.resourceSupportZeroBlocks(destPools, getLinstorRscName(name)); + boolean zeroedDevice = LinstorUtil.resourceSupportZeroBlocks(destPools, getLinstorRscName(dstDisk.getName())); try { final QemuImg qemu = new QemuImg(timeout, zeroedDevice, true); qemu.convert(srcFile, destFile); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 00383c249f88..5766889e697f 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -926,8 +926,7 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) { TemplateInfo tInfo = (TemplateInfo) dstData; final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); final DevelopersApi api = getLinstorAPI(pool); - final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); - boolean newCreated = LinstorUtil.createResourceBase( + Pair crResult = LinstorUtil.createResourceBase( LinstorUtil.RSC_PREFIX + dstData.getUuid(), tInfo.getSize(), tInfo.getName(), @@ -941,7 +940,9 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) { false); Answer answer; - if (newCreated) { + boolean newRscCreated = crResult.second(); + if (newRscCreated) { + String newRscName = crResult.first(); int nMaxExecutionMinutes = NumbersUtil.parseInt( _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); CopyCommand cmd = new CopyCommand( @@ -951,16 +952,16 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) { VirtualMachineManager.ExecuteInSequence.value()); try { - Optional optEP = getLinstorEP(api, pool, rscName); + Optional optEP = getLinstorEP(api, pool, newRscName); if (optEP.isPresent()) { answer = optEP.get().sendMessage(cmd); } else { - deleteResourceDefinition(pool, rscName); + deleteResourceDefinition(pool, newRscName); throw new CloudRuntimeException("Unable to get matching Linstor endpoint."); } } catch (ApiException exc) { logger.error("copy template failed: ", exc); - deleteResourceDefinition(pool, rscName); + deleteResourceDefinition(pool, newRscName); throw new CloudRuntimeException(exc.getBestMessage()); } } else { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 19e443bba2f3..95f6250e0e90 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -381,12 +381,12 @@ public static String getDevicePath(DevelopersApi api, String rscName) throws Api null, null); for (ResourceWithVolumes rsc : resources) { - if (!rsc.getVolumes().isEmpty()) { + if (!rsc.getVolumes().isEmpty() && rsc.getVolumes().get(0).getDevicePath() != null) { return LinstorUtil.getDevicePathFromResource(rsc); } } - final String errMsg = "viewResources didn't return resources or volumes for " + rscName; + final String errMsg = "viewResources didn't return resources or volumes for " + rscName + " with a device path"; LOGGER.error(errMsg); throw new CloudRuntimeException("Linstor: " + errMsg); } @@ -724,7 +724,7 @@ private static void spawnResource( * @param isTemplate indicates if the resource is a template * @return true if a new resource was created, false if it already existed or was reused. */ - public static boolean createResourceBase( + public static Pair createResourceBase( String rscName, long sizeInBytes, String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, String rscGrp, long poolId, boolean isTemplate, boolean exactSize) @@ -740,14 +740,14 @@ public static boolean createResourceBase( existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); if (!alreadyCreated) { boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); + String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; if (createNewRsc) { - String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, volName, vmName, passPhraseId, passPhrase, exactSize); } - return createNewRsc; + return new Pair<>(newRscName, createNewRsc); } - return false; + return new Pair<>(rscName, false); } catch (ApiException apiEx) { LOGGER.error("Linstor: ApiEx - {}", apiEx.getMessage()); From fa3021b5b0654fa4ea5c82c7271ae42b159912d4 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:42:12 +0200 Subject: [PATCH 02/14] linstor: update java-linstor to 0.8.1 Client for LINSTOR REST API 1.29.1 (shipped with linstor-server 1.35.0): make-available gained the auto_manage_dual_primary option, a new unmake-available endpoint was added, resourceSnapshotDelete grew a delete_empty_resource_definition parameter and the resource-definition clone request accepts volume_sizes. Adapt the snapshot delete call sites to the new signature (passing null keeps the previous behavior). --- .../datastore/driver/LinstorPrimaryDataStoreDriverImpl.java | 2 +- .../cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 5766889e697f..3deac390768e 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -254,7 +254,7 @@ private void deleteSnapshot(@Nonnull DataStore dataStore, @Nonnull String rscDef try { - ApiCallRcList answers = linstorApi.resourceSnapshotDelete(rscDefName, snapshotName, Collections.emptyList()); + ApiCallRcList answers = linstorApi.resourceSnapshotDelete(rscDefName, snapshotName, Collections.emptyList(), null); if (answers.hasError()) { for (ApiCallRc answer : answers) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java index d0f82484694b..ee3e17b64090 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/snapshot/LinstorVMSnapshotStrategy.java @@ -214,7 +214,7 @@ private StrategyPriority allVolumesOnLinstor(Long vmId) { private String linstorDeleteSnapshot(final DevelopersApi api, final String rscName, final String snapshotName) { String resultMsg = null; try { - ApiCallRcList answers = api.resourceSnapshotDelete(rscName, snapshotName, Collections.emptyList()); + ApiCallRcList answers = api.resourceSnapshotDelete(rscName, snapshotName, Collections.emptyList(), null); if (answers.hasError()) { resultMsg = LinstorUtil.getBestErrorMessage(answers); } diff --git a/pom.xml b/pom.xml index 1c7ae233d789..751212a7df99 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ 10.1 2.6.6 0.6.0 - 0.7.0 + 0.8.1 0.10.2 3.4.4_1 4.0.1 From 70f114a4fba16def7db7222a90bc06c69f52fe50 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:42:52 +0200 Subject: [PATCH 03/14] linstor: use make-available/unmake-available for live migration LINSTOR REST API 1.29.0 moves the live-migration preparation into the controller: make-available with auto_manage_dual_primary sets DRBD allow-two-primaries (and protocol C) between the migration source and destination, or activates shared storage pool resources on both nodes; unmake-available reverts this on the source node and deletes diskless or redundant shared copies while keeping tiebreaker and diskful resources. connectPhysicalDisk now issues a single make-available with auto_manage_dual_primary set for migrations, and tryDisconnectLinstor a single unmake-available. The controller REST API version is queried per operation (one GET against the controller); against controllers older than 1.29 the previous manual dual-primary, activate and delete handling is used unchanged. --- .../kvm/storage/LinstorStorageAdaptor.java | 71 ++++++++++++++----- .../storage/datastore/util/LinstorUtil.java | 43 +++++++++++ .../datastore/util/LinstorUtilTest.java | 39 ++++++++++ 3 files changed, 135 insertions(+), 18 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 421adb7d6898..590ecbc27a8c 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -365,29 +365,40 @@ public boolean connectPhysicalDisk( } final DevelopersApi api = getLinstorAPI(pool); + final boolean liveMigrateApi = LinstorUtil.supportsLiveMigrateApi(api); String rscName; - boolean usesDrbd; + boolean usesDrbd = false; try { rscName = findCorrectTemplateName(api, getLinstorRscName(volumePath), pool); - usesDrbd = usesDrbd(api, rscName); - - if (isVMMigration && !usesDrbd) { - // create the resource on the new node, noop if already exists - ResourceCreate rc = new ResourceCreate(); - Resource rsc = new Resource(); - rsc.setNodeName(localNodeName); - rc.setResource(rsc); - ApiCallRcList answers = api.resourceCreateOnNode(rscName, localNodeName, rc); - checkLinstorAnswersThrow(answers); - // activate resource if there already was an inactive resource here - answers = api.activateRsc(rscName, localNodeName); - checkLinstorAnswersThrow(answers); - } else { - // DRBD resources just work with a diskless and don't have the activate/deactivate concept + + if (liveMigrateApi) { + // controller >= 1.29 prepares the live migration itself: + // dual-primary for DRBD, activation on both nodes for shared storage pools ResourceMakeAvailable rma = new ResourceMakeAvailable(); + rma.autoManageDualPrimary(isVMMigration); ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); checkLinstorAnswersThrow(answers); + } else { + usesDrbd = usesDrbd(api, rscName); + + if (isVMMigration && !usesDrbd) { + // create the resource on the new node, noop if already exists + ResourceCreate rc = new ResourceCreate(); + Resource rsc = new Resource(); + rsc.setNodeName(localNodeName); + rc.setResource(rsc); + ApiCallRcList answers = api.resourceCreateOnNode(rscName, localNodeName, rc); + checkLinstorAnswersThrow(answers); + // activate resource if there already was an inactive resource here + answers = api.activateRsc(rscName, localNodeName); + checkLinstorAnswersThrow(answers); + } else { + // DRBD resources just work with a diskless and don't have the activate/deactivate concept + ResourceMakeAvailable rma = new ResourceMakeAvailable(); + ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); + checkLinstorAnswersThrow(answers); + } } } catch (ApiException apiEx) { @@ -395,7 +406,7 @@ public boolean connectPhysicalDisk( throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } - if (isVMMigration && usesDrbd) { + if (!liveMigrateApi && isVMMigration && usesDrbd) { try { allow2PrimariesIfInUse(api, rscName); } catch (ApiException apiEx) { @@ -442,6 +453,28 @@ private void removeTwoPrimariesProps(DevelopersApi api, String inUseNode, String removeTwoPrimariesRcProps(api, rscName, inUseNode, deleteProps); } + /** + * Revert a make-available after e.g. a live migration: removes dual-primary settings and + * deletes the resource on this node if that is possible without losing data + * (diskless or a redundant shared storage pool copy; tiebreaker and diskful resources are kept). + * No-op if the resource isn't deployed here, so it is safe to call on every disconnect. + */ + private void unmakeAvailable(DevelopersApi api, String rscName) { + try { + ApiCallRcList answers = api.resourceUnmakeAvailableOnNode(rscName, localNodeName); + if (answers.hasError()) { + // e.g. FAIL_IN_USE: the resource is still in use on this node and is kept + logger.warn("Linstor: unmake-available {} on {}: {}", + rscName, localNodeName, LinstorUtil.getBestErrorMessage(answers)); + } else { + logLinstorAnswers(answers); + } + } catch (ApiException apiEx) { + logger.error(apiEx.getBestMessage()); + // do not fail here, cleaning up after a disconnect isn't fatal + } + } + private void deleteUnusedSharedResources(DevelopersApi api, Resource nodeRsc) { try { List rscs = api.resourceList(nodeRsc.getName(), null, null); @@ -486,9 +519,11 @@ private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) if (optRsc.isPresent()) { Resource rsc = optRsc.get(); + if (LinstorUtil.supportsLiveMigrateApi(api)) { + unmakeAvailable(api, rsc.getName()); // if it is a non DRBD resource(shared) we need to check if we are the last active one // on live migrate, we can have 2 active resources at the same time - if (rsc.getLayerObject().getDrbd() == null) { + } else if (rsc.getLayerObject().getDrbd() == null) { deleteUnusedSharedResources(api, rsc); } else { try { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 95f6250e0e90..8f80bfe819ac 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -103,6 +103,49 @@ public static DevelopersApi getLinstorAPI(String linstorUrl, String apiToken, bo return new DevelopersApi(client); } + /** + * The REST API version of the connected controller, e.g. "1.29.1", or null if it could not be queried. + */ + @Nullable + public static String getRestApiVersion(DevelopersApi api) { + try { + return api.controllerVersion().getRestApiVersion(); + } catch (ApiException apiExc) { + LOGGER.warn("Unable to query controller API version: {}", apiExc.getBestMessage()); + return null; + } + } + + /** + * Check if the connected controller supports the live-migration attach/detach API: + * make-available with auto_manage_dual_primary and unmake-available, added with REST API 1.29.0. + */ + public static boolean supportsLiveMigrateApi(DevelopersApi api) { + return isVersionAtLeast(getRestApiVersion(api), 1, 29, 0); + } + + static boolean isVersionAtLeast(String version, int major, int minor, int patch) { + if (version == null || version.isEmpty()) { + return false; + } + String[] parts = version.split("\\."); + try { + int maj = Integer.parseInt(parts[0]); + int min = parts.length > 1 ? Integer.parseInt(parts[1]) : 0; + int pat = parts.length > 2 ? Integer.parseInt(parts[2]) : 0; + if (maj != major) { + return maj > major; + } + if (min != minor) { + return min > minor; + } + return pat >= patch; + } catch (NumberFormatException nfExc) { + LOGGER.warn("Unable to parse controller API version '{}'", version); + return false; + } + } + public static String getBestErrorMessage(ApiCallRcList answers) { return answers != null && !answers.isEmpty() ? answers.stream() diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index fc4153e33644..0d2667ea7347 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -20,6 +20,7 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.ControllerVersion; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; @@ -190,6 +191,44 @@ public void testDiskfulStoragePoolsIgnoresDiskless() throws ApiException { Assert.assertNull(LinstorUtil.getDiskfulStoragePool(api, "cs-test")); } + @Test + public void testIsVersionAtLeast() { + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.1", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.2", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.30.0", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("2.0.0", 1, 29, 1)); + Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.30", 1, 29, 1)); + + Assert.assertFalse(LinstorUtil.isVersionAtLeast("1.29.0", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("1.29", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("1.28.5", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("0.99.9", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast(null, 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("", 1, 29, 1)); + Assert.assertFalse(LinstorUtil.isVersionAtLeast("garbage", 1, 29, 1)); + } + + private ControllerVersion controllerVersion(String restApiVersion) { + ControllerVersion version = new ControllerVersion(); + version.setRestApiVersion(restApiVersion); + return version; + } + + @Test + public void testSupportsLiveMigrateApi() throws ApiException { + DevelopersApi newCtrl = mock(DevelopersApi.class); + when(newCtrl.controllerVersion()).thenReturn(controllerVersion("1.29.0")); + Assert.assertTrue(LinstorUtil.supportsLiveMigrateApi(newCtrl)); + + DevelopersApi oldCtrl = mock(DevelopersApi.class); + when(oldCtrl.controllerVersion()).thenReturn(controllerVersion("1.28.3")); + Assert.assertFalse(LinstorUtil.supportsLiveMigrateApi(oldCtrl)); + + DevelopersApi unreachable = mock(DevelopersApi.class); + when(unreachable.controllerVersion()).thenThrow(new ApiException(503, "unavailable")); + Assert.assertFalse(LinstorUtil.supportsLiveMigrateApi(unreachable)); + } + @Test public void testGetRscGroupStoragePools() throws ApiException { List storagePools = LinstorUtil.getRscGroupStoragePools(api, "cloudstack"); From 15fd2d2711e3bbdd9b3d452cab4ff7e5fe6db216 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 21 Oct 2026 12:55:38 +0200 Subject: [PATCH 04/14] linstor: support snapshots on thick LVM (shared) storage pools - getSnapshotPath: thick LVM snapshot LVs use the same naming scheme as thin ones - backup: thick LVM snapshot LVs of inactive origins are inactive and carry the skip-activation flag; activate them (lvchange -ay -K) for the copy to secondary storage and deactivate them again afterwards --- .../LinstorBackupSnapshotCommandWrapper.java | 23 +++++++++++++++++++ .../storage/datastore/util/LinstorUtil.java | 1 + .../datastore/util/LinstorUtilTest.java | 11 +++++++++ 3 files changed, 35 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java index 1c48eb2b22f4..9b6772814a76 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java @@ -66,6 +66,19 @@ private String zfsSnapdev(boolean hide, String zfsUrl) { return script.execute(); } + private String lvmSetActive(boolean activate, String devMapperPath) { + // lvm resolves /dev/mapper/vg-lv names textually, works also for inactive LVs + Script script = new Script("lvchange", Duration.millis(5000)); + if (activate) { + script.add("-ay"); + script.add("-K"); // snapshot LVs carry the skip-activation flag + } else { + script.add("-an"); + } + script.add(devMapperPath); + return script.execute(); + } + private String qemuShrink(String path, long sizeByte, long timeout) { Script qemuImg = new Script("qemu-img", Duration.millis(timeout)); qemuImg.add("resize"); @@ -176,6 +189,7 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); KVMStoragePool linstorPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.Linstor, src.getDataStore().getUuid()); boolean zfsHidden = false; + String lvmSnapDev = null; // vg/lv we activated for the copy, deactivated on cleanup String srcPath = src.getPath(); if (linstorPool == null) { @@ -199,6 +213,12 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR return new CopyCmdAnswer("Unable to unhide zfs snapshot device."); } srcPath = "/dev/zvol/" + srcPath.substring(6); + } else if (srcPath.startsWith("/dev/mapper/") && !new File(srcPath).exists()) { + // thick LVM snapshot LVs are created inactive with the skip-activation flag + if (lvmSetActive(true, srcPath) != null) { + return new CopyCmdAnswer("Unable to activate LVM snapshot device " + srcPath); + } + lvmSnapDev = srcPath; } secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl()); @@ -262,6 +282,9 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR if (zfsHidden) { zfsSnapdev(true, src.getPath()); } + if (lvmSnapDev != null) { + lvmSetActive(false, lvmSnapDev); + } } } } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 8f80bfe819ac..f3a7bc9b0067 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -273,6 +273,7 @@ public static String getSnapshotPath(com.linbit.linstor.api.model.StoragePool sp final String backingPool = sp.getProps().get("StorDriver/StorPoolName"); final String path; switch (sp.getProviderKind()) { + case LVM: // thick LVM snapshot LVs use the same naming scheme as thin case LVM_THIN: // device-mapper doubles every dash in the VG and LV name, so the VG part needs escaping too final String vgName = backingPool.split("/")[0]; diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index 0d2667ea7347..6406c5a4d6a1 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -122,6 +122,17 @@ public void testGetSnapshotPath() { snapPath); } + { + StoragePool spLVM = new StoragePool(); + Properties lvmProps = new Properties(); + lvmProps.put("StorDriver/StorPoolName", "shared"); + spLVM.setProps(lvmProps); + spLVM.setProviderKind(ProviderKind.LVM); + String snapPath = LinstorUtil.getSnapshotPath(spLVM, "cs-cb32532a-dd8f-47e0-a81c-8a75573d3545", "cs-6c6b4e95"); + Assert.assertEquals( + "/dev/mapper/shared-cs--cb32532a--dd8f--47e0--a81c--8a75573d3545_00000_cs--6c6b4e95", snapPath); + } + { StoragePool spZFS = new StoragePool(); Properties zfsProps = new Properties(); From ac1a20ec539b7381f129a2231488badf5e124ab9 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:43:49 +0200 Subject: [PATCH 05/14] linstor: activate inactive resources for snapshot backup and revert Shared storage pool resources of stopped VMs are INACTIVE and expose no device on any node. Activate them through the Linstor controller (which takes the shared-space activation lock) before sending the backup or revert copy command to the host and deactivate them again afterwards. The agent itself must only ever activate the snapshot LV, never the resource, to not bypass the shared-space locking. --- .../LinstorPrimaryDataStoreDriverImpl.java | 69 ++++++++++++++----- .../storage/datastore/util/LinstorUtil.java | 14 ++++ 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 3deac390768e..85f541e9c96c 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -664,18 +664,19 @@ private String revertSnapshotFromImageStore( VirtualMachineManager.ExecuteInSequence.value()); final StoragePool pool = (StoragePool) volumeInfo.getDataStore(); - Optional optEP = getDiskfullEP(linstorApi, pool, rscName); - if (optEP.isEmpty()) { - optEP = getLinstorEP(linstorApi, pool, rscName); - } - - if (optEP.isPresent()) { - Answer answer = optEP.get().sendMessage(cmd); - if (!answer.getResult()) { - resultMsg = answer.getDetails(); - } + Host diskfulHost = getDiskfullHost(linstorApi, pool, rscName); + Answer answer; + if (diskfulHost != null) { + answer = sendWithRscActivated(linstorApi, rscName, diskfulHost, cmd); } else { - resultMsg = "Unable to get matching Linstor endpoint."; + Optional optEP = getLinstorEP(linstorApi, pool, rscName); + if (optEP.isEmpty()) { + return "Unable to get matching Linstor endpoint."; + } + answer = optEP.get().sendMessage(cmd); + } + if (!answer.getResult()) { + resultMsg = answer.getDetails(); } return resultMsg; } @@ -858,7 +859,7 @@ private Optional getLinstorEP(DevelopersApi api, StoragePool return Optional.empty(); } - private Optional getDiskfullEP(DevelopersApi api, StoragePool storagePool, String rscName) + private Host getDiskfullHost(DevelopersApi api, StoragePool storagePool, String rscName) throws ApiException { // Take the first diskful copy whose host can be used: the copies are ordered by how // suited they are (in use, then active, then inactive), and for shared storage pools @@ -867,13 +868,46 @@ private Optional getDiskfullEP(DevelopersApi api, StoragePoo : LinstorUtil.getDiskfulStoragePoolsByPreference(api, rscName)) { Host host = getEnabledClusterHost(storagePool, Collections.singletonList(linSP.getNodeName())); if (host != null) { - return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host)); + return host; } logger.debug("Linstor: no usable cloudstack host for diskful node {}, trying next copy", linSP.getNodeName()); } logger.error("Linstor: No diskfull host found."); - return Optional.empty(); + return null; + } + + /** + * Send the given command to the host, temporarily activating the resource there if it is + * inactive (e.g. a shared storage pool resource of a stopped VM). Activation goes through + * the Linstor controller, so the shared-space activation lock is honored; the agent itself + * must only activate/deactivate the snapshot LV, never the resource. + */ + private Answer sendWithRscActivated(DevelopersApi api, String rscName, Host host, CopyCommand cmd) + throws ApiException { + boolean activated = false; + if (LinstorUtil.isResourceInactiveOnNode(api, rscName, host.getName())) { + ApiCallRcList answers = api.activateRsc(rscName, host.getName()); + if (answers.hasError()) { + throw new CloudRuntimeException(String.format( + "Linstor: Unable to activate resource %s on node %s: %s", + rscName, host.getName(), LinstorUtil.getBestErrorMessage(answers))); + } + activated = true; + } + try { + return RemoteHostEndPoint.getHypervisorHostEndPoint(host).sendMessage(cmd); + } finally { + if (activated) { + ApiCallRcList answers = api.deactivateRsc(rscName, host.getName()); + if (answers.hasError()) { + // don't fail the copy over cleanup, but a stray active resource on a shared + // storage pool must be visible in the logs + logger.error("Linstor: Unable to deactivate resource {} on node {}: {}", + rscName, host.getName(), LinstorUtil.getBestErrorMessage(answers)); + } + } + } } private String restoreResourceFromSnapshot( @@ -1116,11 +1150,10 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) { VirtualMachineManager.ExecuteInSequence.value()); cmd.setOptions(options); - Optional optEP = encrypted ? - Optional.empty() : getDiskfullEP(api, pool, rscName); + Host diskfulHost = encrypted ? null : getDiskfullHost(api, pool, rscName); Answer answer; - if (optEP.isPresent()) { - answer = optEP.get().sendMessage(cmd); + if (diskfulHost != null) { + answer = sendWithRscActivated(api, rscName, diskfulHost, cmd); } else { logger.debug("No diskfull endpoint used to copy image (encrypted={}), using temporary resource", encrypted); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index f3a7bc9b0067..001843bc92d3 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -375,6 +375,20 @@ public static String isResourceInUse(DevelopersApi api, String rscName) throws A return null; } + /** + * Check if the resource is deployed but inactive on the given node (e.g. a shared + * storage pool resource of a stopped VM). + */ + public static boolean isResourceInactiveOnNode(DevelopersApi api, String rscName, String nodeName) + throws ApiException { + List rscs = api.resourceList(rscName, null, null); + if (rscs == null) { + return false; + } + return rscs.stream().anyMatch(rsc -> nodeName.equalsIgnoreCase(rsc.getNodeName()) && + rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_RSC_INACTIVE)); + } + /** * Check if the given resources are diskless. * From b161588c27059e0b609ae16c4768b5f0b845e32a Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Fri, 31 Jul 2026 12:44:36 +0200 Subject: [PATCH 06/14] linstor: return expected device path for inactive shared resources The KVM start flow builds the domain XML (createVbd) before connectPhysicalDisk runs make-available, so getDevicePath must not fail for resources that are inactive on every node - the start would spin on the missing device path and never reach the activating connect. Return the deterministic storage layer path instead; make-available activates the resource before the domain is started. Also add the plugin CHANGELOG entries for shared storage pool support. --- plugins/storage/volume/linstor/CHANGELOG.md | 14 ++++++++++++++ .../storage/datastore/util/LinstorUtil.java | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 3f0ef3fc417d..5c36d096056e 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -24,6 +24,20 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2026-07-31] + +### Added + +- Support for shared (thick LVM) storage pools: resources on a shared LUN are + activated on at most one node; live migration uses the LINSTOR 1.29 + make-available/unmake-available API (with fallback to the previous handling + for older controllers), snapshots work on thick LVM pools, and resources of + stopped VMs are activated on demand for start, snapshot backup and revert + +### Changed + +- java-linstor client updated to 0.8.0 (LINSTOR REST API 1.29.0) + ## [2026-07-30] ### Added diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 001843bc92d3..33a18b6be94c 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -444,6 +444,17 @@ public static String getDevicePath(DevelopersApi api, String rscName) throws Api } } + // Shared storage pool resources of stopped VMs are INACTIVE and report no device path. + // Callers like createVbd build the domain XML before connectPhysicalDisk activates the + // resource, so return the deterministic storage layer path here. + com.linbit.linstor.api.model.StoragePool sp = getDiskfulStoragePool(api, rscName); + if (sp != null && (sp.getProviderKind() == ProviderKind.LVM || sp.getProviderKind() == ProviderKind.LVM_THIN)) { + final String vg = sp.getProps().get("StorDriver/StorPoolName").split("/")[0]; + final String path = String.format("/dev/%s/%s_00000", vg, rscName); + LOGGER.info("Linstor: no active device path for {}, using expected LVM path {}", rscName, path); + return path; + } + final String errMsg = "viewResources didn't return resources or volumes for " + rscName + " with a device path"; LOGGER.error(errMsg); throw new CloudRuntimeException("Linstor: " + errMsg); From b9ce72df2bb3f5855efa8575c79b949492dd31ac Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Mon, 3 Aug 2026 20:47:31 +0200 Subject: [PATCH 07/14] linstor: don't register backup snapshot LVs with dmeventd Activating a thick LVM snapshot LV for the backup copy registers it with dmeventd by default; that registration can block lvchange indefinitely (observed stuck in pipe_read waiting for a dmeventd reply), and a blocked lvm command holds the node-wide LVM lock. Pass --monitor n on the temporary activation and deactivation: monitoring only exists to auto-extend snapshots, and Linstor creates thick snapshot LVs at full origin size, so they can never overflow. Also raise the command timeout. --- .../wrapper/LinstorBackupSnapshotCommandWrapper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java index 9b6772814a76..62f4d3da1b6d 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java @@ -68,13 +68,18 @@ private String zfsSnapdev(boolean hide, String zfsUrl) { private String lvmSetActive(boolean activate, String devMapperPath) { // lvm resolves /dev/mapper/vg-lv names textually, works also for inactive LVs - Script script = new Script("lvchange", Duration.millis(5000)); + Script script = new Script("lvchange", Duration.millis(30000)); if (activate) { script.add("-ay"); script.add("-K"); // snapshot LVs carry the skip-activation flag } else { script.add("-an"); } + // never talk to dmeventd: registering the snapshot for monitoring can block lvchange + // indefinitely (and with it the whole LVM lock on the node); a full-size thick COW + // snapshot cannot overflow, so monitoring is not needed for the backup window + script.add("--monitor"); + script.add("n"); script.add(devMapperPath); return script.execute(); } From fe36ba5a0d7b58c992a2bc50789a4ba9aead21a9 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Mon, 3 Aug 2026 21:04:49 +0200 Subject: [PATCH 08/14] linstor: activate inactive template resources for cloning Shared storage pool templates are INACTIVE while no clone is running and Linstor's clone source selection skips inactive resources, failing with 'No suitable storage pools found for cloning'. Activate a diskful template resource for the duration of the clone and deactivate it again afterwards. --- .../LinstorPrimaryDataStoreDriverImpl.java | 10 +++++++ .../storage/datastore/util/LinstorUtil.java | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 85f541e9c96c..293919089d6d 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -439,6 +439,16 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); } } + // Shared storage pool templates are INACTIVE while unused (e.g. after a node + // reboot) and the clone source selection skips inactive resources. Activate one + // diskful resource and leave it active: concurrent clones share the source, so + // deactivating after a clone would race with other clones still using it. + final String activateNode = LinstorUtil.getDiskfulNodeToActivate(linstorApi, cloneRes); + if (activateNode != null) { + logger.info("Linstor: activating template resource {} on {} for cloning", + cloneRes, activateNode); + LinstorUtil.checkLinstorAnswersThrow(linstorApi.activateRsc(cloneRes, activateNode)); + } ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone( cloneRes, cloneRequest); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 33a18b6be94c..6575f4c81dd8 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -375,6 +375,32 @@ public static String isResourceInUse(DevelopersApi api, String rscName) throws A return null; } + /** + * If the resource has no active diskful deployment, return a node with an inactive + * diskful resource that could be activated (e.g. an unused template on a shared + * storage pool). Returns null if an active diskful resource exists or none is deployed. + */ + public static String getDiskfulNodeToActivate(DevelopersApi api, String rscName) throws ApiException { + List rscs = api.resourceList(rscName, null, null); + if (rscs == null) { + return null; + } + String inactiveDiskfulNode = null; + for (Resource rsc : rscs) { + List flags = rsc.getFlags(); + boolean diskless = flags != null && + (flags.contains(ApiConsts.FLAG_DISKLESS) || flags.contains(ApiConsts.FLAG_DRBD_DISKLESS)); + if (diskless) { + continue; + } + if (flags == null || !flags.contains(ApiConsts.FLAG_RSC_INACTIVE)) { + return null; // active diskful resource exists + } + inactiveDiskfulNode = rsc.getNodeName(); + } + return inactiveDiskfulNode; + } + /** * Check if the resource is deployed but inactive on the given node (e.g. a shared * storage pool resource of a stopped VM). From 72b5d1fb483fd2c839b76ac139cd36867d835e8a Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Tue, 4 Aug 2026 13:45:09 +0200 Subject: [PATCH 09/14] kvm: connect source volume for copy to secondary storage copyVolumeFromPrimaryToSecondary read the source device without connecting it first. Detached volumes being migrated between pools are not connected anywhere, and storage drivers that expose devices on demand (e.g. Linstor shared storage pools, where resources of stopped VMs are inactive without a device path) fail the copy. Connect the source volume for the duration of the copy, mirroring what VM start and attach flows do. --- .../cloud/hypervisor/kvm/storage/KVMStorageProcessor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 1fba9f3e96f0..5ad4d5d04a32 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -658,10 +658,14 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) { final String secondaryStorageUrl = nfsStore.getUrl(); KVMStoragePool secondaryStoragePool = null; + boolean srcConnected = false; try { final String volumeName = UUID.randomUUID().toString(); final String destVolumeName = volumeName + "." + ImageFormat.QCOW2.getFileExtension(); + // connect: a detached source volume may have no device yet on shared pools + srcConnected = storagePoolMgr.connectPhysicalDisk( + primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath, null); final KVMPhysicalDisk volume = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath); volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString())); @@ -680,6 +684,9 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) { } finally { srcVol.clearPassphrase(); destVol.clearPassphrase(); + if (srcConnected) { + storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath); + } if (secondaryStoragePool != null) { storagePoolMgr.deleteStoragePool(secondaryStoragePool.getType(), secondaryStoragePool.getUuid()); } From fc10800a73b70f3efd7d621ff10263aa761f8b2c Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Tue, 4 Aug 2026 14:09:54 +0200 Subject: [PATCH 10/14] linstor: pin clone target layer stack to the resource group Clone requests without an explicit layer list can produce a target with a default DRBD layer stack while the source on a shared storage pool is STORAGE only, failing with 'no common clone strategy found'. Set the resource group's configured layer stack on the clone request for unencrypted volumes, mirroring what is already done for encrypted ones. --- .../LinstorPrimaryDataStoreDriverImpl.java | 9 +++++++ .../storage/datastore/util/LinstorUtil.java | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 293919089d6d..3f4e98609100 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -438,6 +438,15 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8); cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); } + } else { + // pin the clone target to the resource group's layer stack (e.g. STORAGE on + // shared pools); otherwise the target can end up with a default DRBD stack + // that has no common clone strategy with the source + List rgLayers = LinstorUtil.getRscGrpLayerList( + linstorApi, LinstorUtil.getRscGrp(storagePoolVO)); + if (!rgLayers.isEmpty()) { + cloneRequest.setLayerList(rgLayers); + } } // Shared storage pool templates are INACTIVE while unused (e.g. after a node // reboot) and the clone source selection skips inactive resources. Activate one diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 6575f4c81dd8..a5a645ea2e1c 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -733,6 +733,30 @@ private static boolean foundShareableTemplate( * @param resourceGroup Resource group to get the encryption layer list * @return layer list with LUKS added */ + /** + * The layer stack configured on the resource group, empty if none is set. + */ + public static List getRscGrpLayerList(DevelopersApi api, String resourceGroup) { + try { + List rscGrps = api.resourceGroupList( + Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); + + if (CollectionUtils.isEmpty(rscGrps)) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + + List layerStack = rscGrps.get(0).getSelectFilter() != null ? + rscGrps.get(0).getSelectFilter().getLayerStack() : Collections.emptyList(); + return CollectionUtils.isNotEmpty(layerStack) ? + layerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()) : + Collections.emptyList(); + } catch (ApiException apiEx) { + LOGGER.error(apiEx.getBestMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } + public static List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { try { List rscGrps = api.resourceGroupList( From 2538d6e18a3b4de3770080fbe0c9292c582f85c2 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 07:57:35 +0200 Subject: [PATCH 11/14] linstor: spawn resources with the resource group's layer stack Rely on the explicit layer stack instead of the resource group default when spawning unencrypted resources. A template spawned with a DRBD stack cannot be used as a clone source for STORAGE or LUKS,STORAGE resources of a shared storage pool later on (the clone either fails with 'no common clone strategy found' or hangs in state CLONING). --- .../cloudstack/storage/datastore/util/LinstorUtil.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index a5a645ea2e1c..4efdf5708634 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -812,6 +812,16 @@ private static void spawnResource( String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); } + } else { + // spell out the resource group's layer stack instead of relying on the implicit + // default: a template spawned with a DRBD stack cannot be cloned into a + // STORAGE/LUKS resource of a shared storage pool later on + List rgLayers = getRscGrpLayerList(api, rscGrpName); + if (!rgLayers.isEmpty()) { + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setLayerStack(rgLayers.stream().map(LayerType::toString).collect(Collectors.toList())); + rscGrpSpawn.setSelectFilter(asf); + } } Properties props = new Properties(); From 90588d897a32b537d00e3c0f90703abea5ba9fc7 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 10:18:03 +0200 Subject: [PATCH 12/14] linstor: count shared storage only once for capacity Every node accessing a shared space (e.g. a LUN with a shared LVM volume group) reports the full capacity of that space, so summing all storage pools of the resource group multiplied the reported capacity by the number of nodes: a 100 GiB shared LUN on three nodes was reported as 300 GiB (and 600 GiB after the over-provisioning factor). Deduplicate storage pools by their shared space when calculating total, used and free capacity. --- .../kvm/storage/LinstorStorageAdaptor.java | 17 +--- .../storage/datastore/util/LinstorUtil.java | 78 ++++++++++++++----- .../datastore/util/LinstorUtilTest.java | 39 ++++++++++ 3 files changed, 101 insertions(+), 33 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 590ecbc27a8c..8af44da9c853 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -48,7 +48,6 @@ import com.linbit.linstor.api.model.ApiCallRcList; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; -import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; import com.linbit.linstor.api.model.ResourceConnectionModify; import com.linbit.linstor.api.model.ResourceCreate; @@ -57,7 +56,6 @@ import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.ResourceWithVolumes; -import com.linbit.linstor.api.model.StoragePool; import com.linbit.linstor.api.model.VolumeDefinition; import java.io.File; @@ -872,12 +870,7 @@ public long getAvailable(LinstorStoragePool pool) { DevelopersApi linstorApi = getLinstorAPI(pool); final String rscGroupName = pool.getResourceGroup(); try { - List storagePools = LinstorUtil.getRscGroupStoragePools(linstorApi, rscGroupName); - - final long free = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getFreeCapacity() != null ? sp.getFreeCapacity() : 0L).sum() * 1024; // linstor uses KiB - + final long free = LinstorUtil.getFreeCapacityBytes(linstorApi, rscGroupName); logger.debug("Linstor: getAvailable() -> " + free); return free; } catch (ApiException apiEx) { @@ -890,13 +883,7 @@ public long getUsed(LinstorStoragePool pool) { DevelopersApi linstorApi = getLinstorAPI(pool); final String rscGroupName = pool.getResourceGroup(); try { - List storagePools = LinstorUtil.getRscGroupStoragePools(linstorApi, rscGroupName); - - final long used = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null && sp.getFreeCapacity() != null ? - sp.getTotalCapacity() - sp.getFreeCapacity() : 0L) - .sum() * 1024; // linstor uses Kib + final long used = LinstorUtil.getUsedCapacityBytes(linstorApi, rscGroupName); logger.debug("Linstor: getUsed() -> " + used); return used; } catch (ApiException apiEx) { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 4efdf5708634..3ae75e425014 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -46,10 +46,12 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; @@ -60,6 +62,7 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -315,15 +318,62 @@ public static List getRscGroupStoragePools(DevelopersApi api, Strin ); } + /** + * Diskful storage pools that contribute capacity, counting storage shared by multiple + * nodes only once. Nodes accessing the same shared space (e.g. a LUN with a shared LVM + * volume group) all report the full capacity of that space, so summing them as-is + * multiplies the real capacity by the number of nodes. + * + * Deduplication uses the free space manager name: node local storage pools get a + * per-node name ("node;pool"), pools sharing a space all report the shared space name. + */ + public static List getCapacityStoragePools( + @Nonnull DevelopersApi api, String rscGroupName) throws ApiException { + List storagePools = getRscGroupStoragePools(api, rscGroupName); + List distinct = new ArrayList<>(); + Set seenSpaces = new HashSet<>(); + for (StoragePool sp : storagePools) { + if (sp.getProviderKind() == ProviderKind.DISKLESS) { + continue; + } + if (StringUtils.isEmpty(sp.getFreeSpaceMgrName()) || seenSpaces.add(sp.getFreeSpaceMgrName())) { + distinct.add(sp); + } + } + return distinct; + } + + private static long sumTotalCapacity(List storagePools) { + return storagePools.stream() + .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) + .sum() * 1024; // linstor uses KiB + } + + private static long sumUsedCapacity(List storagePools) { + return storagePools.stream() + .mapToLong(sp -> sp.getTotalCapacity() != null && sp.getFreeCapacity() != null ? + sp.getTotalCapacity() - sp.getFreeCapacity() : 0L) + .sum() * 1024; // linstor uses KiB + } + + private static long sumFreeCapacity(List storagePools) { + return storagePools.stream() + .mapToLong(sp -> sp.getFreeCapacity() != null ? sp.getFreeCapacity() : 0L) + .sum() * 1024; // linstor uses KiB + } + + public static long getFreeCapacityBytes(@Nonnull DevelopersApi api, String rscGroupName) throws ApiException { + return sumFreeCapacity(getCapacityStoragePools(api, rscGroupName)); + } + + public static long getUsedCapacityBytes(@Nonnull DevelopersApi api, String rscGroupName) throws ApiException { + return sumUsedCapacity(getCapacityStoragePools(api, rscGroupName)); + } + public static long getCapacityBytes(String linstorUrl, String rscGroupName, String apiToken, boolean insecureSsl) { DevelopersApi linstorApi = getLinstorAPI(linstorUrl, apiToken, insecureSsl); try { - List storagePools = getRscGroupStoragePools(linstorApi, rscGroupName); - - return storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) - .sum() * 1024; // linstor uses kiB + return sumTotalCapacity(getCapacityStoragePools(linstorApi, rscGroupName)); } catch (ApiException apiEx) { LOGGER.error(apiEx.getMessage()); throw new CloudRuntimeException(apiEx); @@ -333,18 +383,10 @@ public static long getCapacityBytes(String linstorUrl, String rscGroupName, Stri public static Pair getStorageStats(String linstorUrl, String rscGroupName, String apiToken, boolean insecureSsl) { DevelopersApi linstorApi = getLinstorAPI(linstorUrl, apiToken, insecureSsl); try { - List storagePools = LinstorUtil.getRscGroupStoragePools(linstorApi, rscGroupName); - - long capacity = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) - .sum() * 1024; // linstor uses kiB - - long used = storagePools.stream() - .filter(sp -> sp.getProviderKind() != ProviderKind.DISKLESS) - .mapToLong(sp -> sp.getTotalCapacity() != null && sp.getFreeCapacity() != null ? - sp.getTotalCapacity() - sp.getFreeCapacity() : 0L) - .sum() * 1024; // linstor uses Kib + List storagePools = getCapacityStoragePools(linstorApi, rscGroupName); + + long capacity = sumTotalCapacity(storagePools); + long used = sumUsedCapacity(storagePools); LOGGER.debug( String.format("Linstor(%s;%s): storageStats -> %d/%d", linstorUrl, rscGroupName, capacity, used)); return new Pair<>(capacity, used); diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index 6406c5a4d6a1..493a493814b7 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -65,6 +65,15 @@ private StoragePool mockStoragePool(String name, String node, ProviderKind kind) return sp; } + private StoragePool mockStoragePool( + String name, String node, ProviderKind kind, String freeSpaceMgrName, long totalKib, long freeKib) { + StoragePool sp = mockStoragePool(name, node, kind); + sp.setFreeSpaceMgrName(freeSpaceMgrName); + sp.setTotalCapacity(totalKib); + sp.setFreeCapacity(freeKib); + return sp; + } + @Before public void setUp() throws ApiException { api = mock(DevelopersApi.class); @@ -202,6 +211,36 @@ public void testDiskfulStoragePoolsIgnoresDiskless() throws ApiException { Assert.assertNull(LinstorUtil.getDiskfulStoragePool(api, "cs-test")); } + @Test + public void testGetCapacityStoragePoolsCountsSharedSpaceOnce() throws ApiException { + ResourceGroup sharedGroup = new ResourceGroup(); + sharedGroup.setName("shared"); + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setPlaceCount(1); + sharedGroup.setSelectFilter(asf); + when(api.resourceGroupList(Collections.singletonList("shared"), null, null, null)) + .thenReturn(Collections.singletonList(sharedGroup)); + + // 3 nodes accessing the same shared space, each reporting the full 100 GiB, plus one + // node-local pool and a diskless pool which must be ignored + when(api.viewStoragePools(Collections.emptyList(), null, null, null, null, true)) + .thenReturn(Arrays.asList( + mockStoragePool("sharedpool", "nodeA", ProviderKind.LVM, "cs-shared", 104857600L, 52428800L), + mockStoragePool("sharedpool", "nodeB", ProviderKind.LVM, "cs-shared", 104857600L, 52428800L), + mockStoragePool("sharedpool", "nodeC", ProviderKind.LVM, "cs-shared", 104857600L, 52428800L), + mockStoragePool("local", "nodeA", ProviderKind.LVM_THIN, "nodeA;local", 10485760L, 10485760L), + mockStoragePool("diskless", "nodeB", ProviderKind.DISKLESS, "nodeB;diskless", 0L, 0L) + )); + + List pools = LinstorUtil.getCapacityStoragePools(api, "shared"); + Assert.assertEquals(2, pools.size()); + // 100 GiB shared (once) + 10 GiB local + Assert.assertEquals((104857600L + 10485760L) * 1024, LinstorUtil.getFreeCapacityBytes(api, "shared") + + LinstorUtil.getUsedCapacityBytes(api, "shared")); + // half of the shared space is used, the local pool is empty + Assert.assertEquals(52428800L * 1024, LinstorUtil.getUsedCapacityBytes(api, "shared")); + } + @Test public void testIsVersionAtLeast() { Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.1", 1, 29, 1)); From 129ba38b19cdf0eaa4ffbab356647eb9ec719c2d Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 10:43:52 +0200 Subject: [PATCH 13/14] linstor: no over-provisioning for thick storage pools Thickly provisioned backends (thick LVM, ZFS with reservation, file) allocate the full volume size, so the pool capacity is a hard limit and the storage.overprovisioning.factor default of 2 makes CloudStack allocate twice the space that exists. The Linstor storage pool type has to allow over-provisioning for the thin backends, so set the factor to 1.0 per pool when all storage pools of the resource group are thick. Mixed or unknown setups keep the default. --- .../LinstorPrimaryDataStoreLifeCycleImpl.java | 10 +++++++ .../storage/datastore/util/LinstorUtil.java | 27 +++++++++++++++++++ .../datastore/util/LinstorUtilTest.java | 14 ++++++++++ 3 files changed, 51 insertions(+) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java index 7c1330b9088e..4883de102159 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java @@ -207,6 +207,16 @@ public DataStore initialize(Map dsInfos) { LinstorConfigurationManager.ApiToken.key(), DBEncryptionUtil.encrypt(apiToken), false); } + // Thickly provisioned storage (e.g. thick LVM, also used for shared storage pools) allocates + // the full volume size, so the pool capacity is a hard limit. The Linstor storage pool type + // allows over-provisioning for thin backends, so disable it per pool here instead. + if (dataStore != null && LinstorUtil.isThicklyProvisioned(url, resourceGroup, apiToken, insecureSsl)) { + logger.info("Linstor: {} is thickly provisioned, setting {} to 1.0", + storagePoolName, CapacityManager.StorageOverprovisioningFactor.key()); + storagePoolDetailsDao.addDetail(dataStore.getId(), + CapacityManager.StorageOverprovisioningFactor.key(), "1.0", true); + } + return dataStore; } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 3ae75e425014..f46d9ad3e3b6 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -343,6 +343,33 @@ public static List getCapacityStoragePools( return distinct; } + /** + * Storage provider kinds that allocate the full volume size up front. Such pools cannot be + * over-provisioned: their capacity is the real limit. + */ + private static final Collection THICK_PROVIDER_KINDS = Arrays.asList( + ProviderKind.LVM, ProviderKind.ZFS, ProviderKind.FILE); + + /** + * Check if all storage pools backing the resource group are thickly provisioned. + * Unknown/mixed setups are reported as thin, keeping the over-provisioning default. + */ + public static boolean isThicklyProvisioned(String linstorUrl, String rscGroupName, String apiToken, + boolean insecureSsl) { + DevelopersApi api = getLinstorAPI(linstorUrl, apiToken, insecureSsl); + try { + return isThicklyProvisionedPools(getCapacityStoragePools(api, rscGroupName)); + } catch (ApiException apiEx) { + LOGGER.warn("Unable to query storage pool provider kinds: {}", apiEx.getBestMessage()); + return false; + } + } + + static boolean isThicklyProvisionedPools(List storagePools) { + return !storagePools.isEmpty() && storagePools.stream() + .allMatch(sp -> THICK_PROVIDER_KINDS.contains(sp.getProviderKind())); + } + private static long sumTotalCapacity(List storagePools) { return storagePools.stream() .mapToLong(sp -> sp.getTotalCapacity() != null ? sp.getTotalCapacity() : 0L) diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index 493a493814b7..4711e4b07ec4 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -241,6 +241,20 @@ public void testGetCapacityStoragePoolsCountsSharedSpaceOnce() throws ApiExcepti Assert.assertEquals(52428800L * 1024, LinstorUtil.getUsedCapacityBytes(api, "shared")); } + @Test + public void testThickProviderKindDetection() { + Assert.assertTrue(LinstorUtil.isThicklyProvisionedPools(Arrays.asList( + mockStoragePool("sharedpool", "nodeA", ProviderKind.LVM, "cs-shared", 1L, 1L), + mockStoragePool("sharedpool", "nodeB", ProviderKind.LVM, "cs-shared", 1L, 1L)))); + Assert.assertFalse(LinstorUtil.isThicklyProvisionedPools(Arrays.asList( + mockStoragePool("thinpool", "nodeA", ProviderKind.LVM_THIN, "nodeA;thinpool", 1L, 1L)))); + // mixed setups keep the over-provisioning default + Assert.assertFalse(LinstorUtil.isThicklyProvisionedPools(Arrays.asList( + mockStoragePool("sharedpool", "nodeA", ProviderKind.LVM, "cs-shared", 1L, 1L), + mockStoragePool("thinpool", "nodeB", ProviderKind.LVM_THIN, "nodeB;thinpool", 1L, 1L)))); + Assert.assertFalse(LinstorUtil.isThicklyProvisionedPools(Collections.emptyList())); + } + @Test public void testIsVersionAtLeast() { Assert.assertTrue(LinstorUtil.isVersionAtLeast("1.29.1", 1, 29, 1)); From c5601058ca0635bc90e1216f6bb2df1e2dbc89e9 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 5 Aug 2026 14:15:45 +0200 Subject: [PATCH 14/14] linstor: only send auto_manage_dual_primary when migrating Linstor controllers older than 1.29 reject requests with unknown properties, so a make-available carrying auto_manage_dual_primary fails against them - which broke every make-available call (and with it template copies and VM starts) even though the live-migrate API is version gated: the generated client initializes fields with their schema default (false here) and serialized everything non-null, so the property was always sent. --- .../cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 8af44da9c853..91e7bce5b9ed 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -374,7 +374,10 @@ public boolean connectPhysicalDisk( // controller >= 1.29 prepares the live migration itself: // dual-primary for DRBD, activation on both nodes for shared storage pools ResourceMakeAvailable rma = new ResourceMakeAvailable(); - rma.autoManageDualPrimary(isVMMigration); + if (isVMMigration) { + // only set when needed: older controllers reject the unknown property + rma.autoManageDualPrimary(true); + } ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, localNodeName, rma); checkLinstorAnswersThrow(answers); } else {