Skip to content

Commit e2d18c0

Browse files
sureshanapartisureshanaparti
committed
Merge branch '4.22'
2 parents 4f93ba8 + 4708121 commit e2d18c0

19 files changed

Lines changed: 262 additions & 29 deletions

File tree

‎core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import com.cloud.utils.Pair;
5353
import com.cloud.utils.UriUtils;
5454
import com.cloud.utils.exception.CloudRuntimeException;
55+
import com.cloud.utils.net.HttpClientCloudStackUserAgent;
5556
import com.cloud.utils.net.Proxy;
5657

5758
/**
@@ -125,6 +126,7 @@ private GetMethod createRequest(String downloadUrl) {
125126
GetMethod request = new GetMethod(downloadUrl);
126127
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
127128
request.setFollowRedirects(followRedirects);
129+
request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
128130
return request;
129131
}
130132

‎core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@
1818
//
1919
package com.cloud.storage.template;
2020

21+
2122
import com.cloud.storage.StorageLayer;
2223
import com.cloud.utils.UriUtils;
24+
import com.cloud.utils.net.HttpClientCloudStackUserAgent;
25+
2326
import org.apache.commons.httpclient.HttpClient;
2427
import org.apache.commons.httpclient.HttpMethod;
2528
import org.apache.commons.httpclient.HttpMethodRetryHandler;
@@ -59,6 +62,7 @@ protected GetMethod createRequest(String downloadUrl) {
5962
GetMethod request = new GetMethod(downloadUrl);
6063
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
6164
request.setFollowRedirects(followRedirects);
65+
request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
6266
if (!toFileSet) {
6367
String[] parts = downloadUrl.split("/");
6468
String filename = parts[parts.length - 1];

‎core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.apache.commons.lang3.StringUtils;
4545

4646
import com.cloud.storage.StorageLayer;
47+
import com.cloud.utils.net.HttpClientCloudStackUserAgent;
4748

4849
public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader {
4950
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
@@ -95,6 +96,7 @@ private GetMethod createRequest(String downloadUrl) {
9596
GetMethod request = new GetMethod(downloadUrl);
9697
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
9798
request.setFollowRedirects(followRedirects);
99+
request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
98100
return request;
99101
}
100102

@@ -141,6 +143,7 @@ private void tryAndGetTotalRemoteSize() {
141143
continue;
142144
}
143145
HeadMethod headMethod = new HeadMethod(downloadUrl);
146+
headMethod.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
144147
try {
145148
if (client.executeMethod(headMethod) != HttpStatus.SC_OK) {
146149
continue;

‎core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.apache.cloudstack.direct.download;
2121

22+
2223
import java.io.File;
2324
import java.io.FileOutputStream;
2425
import java.io.IOException;
@@ -32,13 +33,15 @@
3233
import com.cloud.utils.Pair;
3334
import com.cloud.utils.UriUtils;
3435
import com.cloud.utils.exception.CloudRuntimeException;
36+
import com.cloud.utils.net.HttpClientCloudStackUserAgent;
3537
import com.cloud.utils.storage.QCOW2Utils;
3638
import org.apache.commons.collections.MapUtils;
3739
import org.apache.commons.httpclient.HttpClient;
3840
import org.apache.commons.httpclient.HttpStatus;
3941
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
4042
import org.apache.commons.httpclient.methods.GetMethod;
4143
import org.apache.commons.httpclient.methods.HeadMethod;
44+
import org.apache.commons.httpclient.params.HttpMethodParams;
4245
import org.apache.commons.io.IOUtils;
4346

4447
public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@@ -68,6 +71,7 @@ public HttpDirectTemplateDownloader(String url, Long templateId, String destPool
6871
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
6972
GetMethod request = new GetMethod(downloadUrl);
7073
request.setFollowRedirects(this.isFollowRedirects());
74+
request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
7175
if (MapUtils.isNotEmpty(headers)) {
7276
for (String key : headers.keySet()) {
7377
request.setRequestHeader(key, headers.get(key));
@@ -111,6 +115,7 @@ protected Pair<Boolean, String> performDownload() {
111115
public boolean checkUrl(String url) {
112116
HeadMethod httpHead = new HeadMethod(url);
113117
httpHead.setFollowRedirects(this.isFollowRedirects());
118+
httpHead.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
114119
try {
115120
int responseCode = client.executeMethod(httpHead);
116121
if (responseCode != HttpStatus.SC_OK) {

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package com.cloud.vm;
1919

20+
import static com.cloud.configuration.ConfigurationManagerImpl.EXPOSE_ERRORS_TO_USER;
2021
import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS;
2122

2223
import java.lang.reflect.Field;
@@ -933,10 +934,22 @@ public void start(final String vmUuid, final Map<VirtualMachineProfile.Param, Ob
933934
public void start(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) {
934935
try {
935936
advanceStart(vmUuid, params, planToDeploy, planner);
936-
} catch (ConcurrentOperationException | InsufficientCapacityException e) {
937-
throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid);
937+
} catch (ConcurrentOperationException e) {
938+
final CallContext cctxt = CallContext.current();
939+
final Account account = cctxt.getCallingAccount();
940+
if (canExposeError(account)) {
941+
throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid);
942+
}
943+
throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to concurrent operation.", vmUuid), e).add(VirtualMachine.class, vmUuid);
944+
} catch (final InsufficientCapacityException e) {
945+
final CallContext cctxt = CallContext.current();
946+
final Account account = cctxt.getCallingAccount();
947+
if (canExposeError(account)) {
948+
throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid);
949+
}
950+
throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to insufficient capacity.", vmUuid), e).add(VirtualMachine.class, vmUuid);
938951
} catch (final ResourceUnavailableException e) {
939-
if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)){
952+
if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)) {
940953
Account callingAccount = CallContext.current().getCallingAccount();
941954
String errorSuffix = (callingAccount != null && callingAccount.getType() == Account.Type.ADMIN) ?
942955
String.format("Failure: %s", e.getMessage()) :
@@ -1367,6 +1380,7 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
13671380

13681381
final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType());
13691382

1383+
Throwable lastKnownError = null;
13701384
boolean canRetry = true;
13711385
ExcludeList avoids = null;
13721386
try {
@@ -1390,7 +1404,8 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
13901404

13911405
int retry = StartRetry.value();
13921406
while (retry-- != 0) {
1393-
logger.debug("Instance start attempt #{}", (StartRetry.value() - retry));
1407+
int attemptNumber = StartRetry.value() - retry;
1408+
logger.debug("Instance start attempt #{}", attemptNumber);
13941409

13951410
if (reuseVolume) {
13961411
final List<VolumeVO> vols = _volsDao.findReadyRootVolumesByInstance(vm.getId());
@@ -1456,8 +1471,13 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
14561471
reuseVolume = false;
14571472
continue;
14581473
}
1459-
throw new InsufficientServerCapacityException("Unable to create a deployment for " + vmProfile, DataCenter.class, plan.getDataCenterId(),
1460-
areAffinityGroupsAssociated(vmProfile));
1474+
String message = String.format("Unable to create a deployment for %s after %s attempts", vmProfile, attemptNumber);
1475+
if (canExposeError(account) && lastKnownError != null) {
1476+
message += String.format(" Last known error: %s", lastKnownError.getMessage());
1477+
throw new CloudRuntimeException(message, lastKnownError);
1478+
} else {
1479+
throw new InsufficientServerCapacityException(message, DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile));
1480+
}
14611481
}
14621482

14631483
avoids.addHost(dest.getHost().getId());
@@ -1625,11 +1645,15 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
16251645
throw new ExecutionException("Unable to start VM:" + vm.getUuid() + " due to error in finalizeStart, not retrying");
16261646
}
16271647
}
1628-
logger.info("Unable to start VM on {} due to {}", dest.getHost(), (startAnswer == null ? " no start answer" : startAnswer.getDetails()));
1648+
String msg = String.format("Unable to start VM on %s due to %s", dest.getHost(), startAnswer == null ? "no start command answer" : startAnswer.getDetails());
1649+
lastKnownError = new ExecutionException(msg);
1650+
16291651
if (startAnswer != null && startAnswer.getContextParam("stopRetry") != null) {
1652+
logger.error(msg, lastKnownError);
16301653
break;
16311654
}
16321655

1656+
logger.debug(msg, lastKnownError);
16331657
} catch (OperationTimedoutException e) {
16341658
logger.debug("Unable to send the start command to host {} failed to start VM: {}", dest.getHost(), vm);
16351659
if (e.isActive()) {
@@ -1639,6 +1663,7 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
16391663
throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e);
16401664
} catch (final ResourceUnavailableException e) {
16411665
logger.warn("Unable to contact resource.", e);
1666+
lastKnownError = e;
16421667
if (!avoids.add(e)) {
16431668
if (e.getScope() == Volume.class || e.getScope() == Nic.class) {
16441669
throw e;
@@ -1695,10 +1720,22 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
16951720
}
16961721

16971722
if (startedVm == null) {
1698-
throw new CloudRuntimeException("Unable to start Instance '" + vm.getHostName() + "' (" + vm.getUuid() + "), see management server log for details");
1723+
String messageTmpl = "Unable to start Instance '%s' (%s)%s";
1724+
String details;
1725+
if (canExposeError(account) && lastKnownError != null) {
1726+
details = ": " + lastKnownError.getMessage();
1727+
} else {
1728+
details = ", see management server log for details";
1729+
}
1730+
String message = String.format(messageTmpl, vm.getHostName(), vm.getUuid(), details);
1731+
throw new CloudRuntimeException(message, lastKnownError);
16991732
}
17001733
}
17011734

1735+
private boolean canExposeError(Account account) {
1736+
return (account != null && account.getType() == Account.Type.ADMIN) || Boolean.TRUE.equals(EXPOSE_ERRORS_TO_USER.value());
1737+
}
1738+
17021739
protected void updateStartCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, StartCommand command) {
17031740
if (!HypervisorType.External.equals(host.getHypervisorType())) {
17041741
return;

‎engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ protected void init() {
193193
PersistentNetworkSearch.and("id", PersistentNetworkSearch.entity().getId(), Op.NEQ);
194194
PersistentNetworkSearch.and("guestType", PersistentNetworkSearch.entity().getGuestType(), Op.IN);
195195
PersistentNetworkSearch.and("broadcastUri", PersistentNetworkSearch.entity().getBroadcastUri(), Op.EQ);
196+
PersistentNetworkSearch.and("dc", PersistentNetworkSearch.entity().getDataCenterId(), Op.EQ);
196197
PersistentNetworkSearch.and("removed", PersistentNetworkSearch.entity().getRemoved(), Op.NULL);
197198
final SearchBuilder<NetworkOfferingVO> persistentNtwkOffJoin = _ntwkOffDao.createSearchBuilder();
198199
persistentNtwkOffJoin.and("persistent", persistentNtwkOffJoin.entity().isPersistent(), Op.EQ);

‎engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,6 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU';
3535
UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent';
3636

3737
UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template';
38+
39+
-- Update existing vm_template records with NULL type to "USER"
40+
UPDATE `cloud`.`vm_template` SET `type` = 'USER' WHERE `type` IS NULL;

‎engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
4141
import org.apache.cloudstack.storage.to.VolumeObjectTO;
4242
import org.apache.commons.collections.CollectionUtils;
43+
import org.apache.cloudstack.storage.datastore.api.Volume;
4344

4445
import com.cloud.agent.api.VMSnapshotTO;
4546
import com.cloud.alert.AlertManager;
@@ -200,11 +201,35 @@ public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) {
200201
if (volumeIds != null && !volumeIds.isEmpty()) {
201202
List<VMSnapshotDetailsVO> vmSnapshotDetails = new ArrayList<VMSnapshotDetailsVO>();
202203
vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "SnapshotGroupId", snapshotGroupId, false));
204+
Map<String, String> snapshotNameToSrcPathMap = new HashMap<>();
205+
for (Map.Entry<String, String> entry : srcVolumeDestSnapshotMap.entrySet()) {
206+
snapshotNameToSrcPathMap.put(entry.getValue(), entry.getKey());
207+
}
208+
209+
for (String snapshotVolumeId : volumeIds) {
210+
// Use getVolume() to fetch snapshot volume details and get its name
211+
Volume snapshotVolume = client.getVolume(snapshotVolumeId);
212+
if (snapshotVolume == null) {
213+
throw new CloudRuntimeException("Cannot find snapshot volume with id: " + snapshotVolumeId);
214+
}
215+
String snapshotName = snapshotVolume.getName();
216+
217+
// Match back to source volume path
218+
String srcVolumePath = snapshotNameToSrcPathMap.get(snapshotName);
219+
if (srcVolumePath == null) {
220+
throw new CloudRuntimeException("Cannot match snapshot " + snapshotName + " to a source volume");
221+
}
222+
223+
// Find the matching VolumeObjectTO by path
224+
VolumeObjectTO matchedVolume = volumeTOs.stream()
225+
.filter(v -> ScaleIOUtil.getVolumePath(v.getPath()).equals(srcVolumePath))
226+
.findFirst()
227+
.orElseThrow(() -> new CloudRuntimeException("Cannot find source volume for path: " + srcVolumePath));
203228

204-
for (int index = 0; index < volumeIds.size(); index++) {
205-
String volumeSnapshotName = srcVolumeDestSnapshotMap.get(ScaleIOUtil.getVolumePath(volumeTOs.get(index).getPath()));
206-
String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(volumeIds.get(index), volumeSnapshotName);
207-
vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "Vol_" + volumeTOs.get(index).getId() + "_Snapshot", pathWithScaleIOVolumeName, false));
229+
String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, snapshotName);
230+
vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(),
231+
"Vol_" + matchedVolume.getId() + "_Snapshot",
232+
pathWithScaleIOVolumeName, false));
208233
}
209234

210235
vmSnapshotDetailsDao.saveDetails(vmSnapshotDetails);

‎plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,9 @@ public boolean deleteBackup(Backup backup, boolean forced) {
478478
} else {
479479
host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId());
480480
}
481+
if (host == null) {
482+
throw new CloudRuntimeException(String.format("Unable to find a running KVM host in zone %d to delete backup %s", backup.getZoneId(), backup.getUuid()));
483+
}
481484

482485
DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(),
483486
backupRepository.getAddress(), backupRepository.getMountOptions());
@@ -564,7 +567,14 @@ public Pair<Long, Long> getBackupStorageStats(Long zoneId) {
564567
@Override
565568
public void syncBackupStorageStats(Long zoneId) {
566569
final List<BackupRepository> repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName());
570+
if (CollectionUtils.isEmpty(repositories)) {
571+
return;
572+
}
567573
final Host host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId);
574+
if (host == null) {
575+
logger.warn("Unable to find a running KVM host in zone {} to sync backup storage stats", zoneId);
576+
return;
577+
}
568578
for (final BackupRepository repository : repositories) {
569579
GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions());
570580
BackupStorageStatsAnswer answer;

0 commit comments

Comments
 (0)