PowerFlex/ScaleIO bugfix Jan Patch (#110)
* Addressed some issues for the operations on PowerFlex storage pool.
- Updated VM Snapshot naming, for uniqueness in ScaleIO volume name when more than one volume exists in the VM.
- Added sync lock while spooling managed storage template before volume creation from the template (non-direct download).
- Updated resize volume error message string.
- Blocked the below operations on PowerFlex storage pool:
-> Extract Volume
-> Create Snapshot for VMSnapshot
* Fail the VM deployment when the host specified in the deployVirtualMachine cmd is not in the right state (i.e. either Resource State is not Enabled or Status is not Up)
* Use single gateway client per Powerflex/ScaleIO pool and renew it when the session token expires.
The token is valid for 8 hours from the time it was created, unless there has been no activity for 10 minutes.
Reference: https://cpsdocs.dellemc.com/bundle/PF_REST_API_RG/page/GUID-92430F19-9F44-42B6-B898-87D5307AE59B.html
* Use the physical file size of the template to check the free space availability on the host, while downloading the direct download templates.
* Perform basic tests (for connectivity and file system) on router before updating the health check config data
Updated changes to:
- Validate the basic tests (connectivity and file system check) on router
- Cleanup the health check results when router is destroyed
diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java
index 98fb8be..8504efd 100644
--- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java
+++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java
@@ -26,6 +26,7 @@
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.network.router.VirtualRouter;
import com.cloud.user.Account;
+import com.cloud.utils.Pair;
public interface VirtualNetworkApplianceService {
/**
@@ -73,5 +74,5 @@
* @param routerId id of the router
* @return
*/
- boolean performRouterHealthChecks(long routerId);
+ Pair<Boolean, String> performRouterHealthChecks(long routerId);
}
diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java
index 5efc6de..dc1020b 100644
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java
@@ -111,7 +111,7 @@
setResponseObject(routerResponse);
} catch (CloudRuntimeException ex){
ex.printStackTrace();
- throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to execute command due to exception: " + ex.getLocalizedMessage());
+ throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to get health check results due to: " + ex.getLocalizedMessage());
}
}
}
diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java b/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java
index 779a0f4..e32dda3 100644
--- a/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java
+++ b/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java
@@ -19,12 +19,14 @@
public class GetRouterMonitorResultsCommand extends NetworkElementCommand {
private boolean performFreshChecks;
+ private boolean validateBasicTestsOnly;
protected GetRouterMonitorResultsCommand() {
}
- public GetRouterMonitorResultsCommand(boolean performFreshChecks) {
+ public GetRouterMonitorResultsCommand(boolean performFreshChecks, boolean validateBasicTestsOnly) {
this.performFreshChecks = performFreshChecks;
+ this.validateBasicTestsOnly = validateBasicTestsOnly;
}
@Override
@@ -35,4 +37,8 @@
public boolean shouldPerformFreshChecks() {
return performFreshChecks;
}
+
+ public boolean shouldValidateBasicTestsOnly() {
+ return validateBasicTestsOnly;
+ }
}
\ No newline at end of file
diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java
index 8b32842..395a174 100644
--- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java
+++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java
@@ -66,6 +66,7 @@
import com.cloud.agent.resource.virtualnetwork.facade.AbstractConfigItemFacade;
import com.cloud.utils.ExecutionResult;
import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
/**
@@ -312,20 +313,14 @@
private GetRouterMonitorResultsAnswer execute(GetRouterMonitorResultsCommand cmd) {
String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
- ExecutionResult fsReadOnlyResult = _vrDeployer.executeInVR(routerIp, VRScripts.ROUTER_FILESYSTEM_WRITABLE_CHECK, null);
- if (!fsReadOnlyResult.isSuccess()) {
- s_logger.warn("Result of " + cmd + " failed with details: " + fsReadOnlyResult.getDetails());
- if (StringUtils.isNotBlank(fsReadOnlyResult.getDetails())) {
- final String readOnlyFileSystemError = "Read-only file system";
- if (fsReadOnlyResult.getDetails().contains(readOnlyFileSystemError)) {
- return new GetRouterMonitorResultsAnswer(cmd, false, null, readOnlyFileSystemError);
- } else {
- return new GetRouterMonitorResultsAnswer(cmd, false, null, fsReadOnlyResult.getDetails());
- }
- } else {
- s_logger.warn("Result of " + cmd + " received empty details.");
- return new GetRouterMonitorResultsAnswer(cmd, false, null, "No results available.");
- }
+ Pair<Boolean, String> fileSystemTestResult = checkRouterFileSystem(routerIp);
+ if (!fileSystemTestResult.first()) {
+ return new GetRouterMonitorResultsAnswer(cmd, false, null, fileSystemTestResult.second());
+ }
+
+ if (cmd.shouldValidateBasicTestsOnly()) {
+ // Basic tests (connectivity and file system checks) are already validated
+ return new GetRouterMonitorResultsAnswer(cmd, true, null, "success");
}
String args = cmd.shouldPerformFreshChecks() ? "true" : "false";
@@ -345,6 +340,27 @@
return parseLinesForHealthChecks(cmd, result.getDetails());
}
+ private Pair<Boolean, String> checkRouterFileSystem(String routerIp) {
+ ExecutionResult fileSystemWritableTestResult = _vrDeployer.executeInVR(routerIp, VRScripts.ROUTER_FILESYSTEM_WRITABLE_CHECK, null);
+ if (fileSystemWritableTestResult.isSuccess()) {
+ s_logger.debug("Router connectivity and file system writable check passed");
+ return new Pair<Boolean, String>(true, "success");
+ }
+
+ String resultDetails = fileSystemWritableTestResult.getDetails();
+ s_logger.warn("File system writable check failed with details: " + resultDetails);
+ if (StringUtils.isNotBlank(resultDetails)) {
+ final String readOnlyFileSystemError = "Read-only file system";
+ if (resultDetails.contains(readOnlyFileSystemError)) {
+ resultDetails = "Read-only file system";
+ }
+ } else {
+ resultDetails = "No results available";
+ }
+
+ return new Pair<Boolean, String>(false, resultDetails);
+ }
+
private GetRouterAlertsAnswer execute(GetRouterAlertsCommand cmd) {
String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
index 316926a..13eccd7 100644
--- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
+++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
@@ -122,7 +122,7 @@
"storage.pool.client.timeout",
"Storage",
"60",
- "Timeout (in secs) for the storage pool client timeout (for managed pools). Currently only supported for PowerFlex.",
+ "Timeout (in secs) for the storage pool client connection timeout (for managed pools). Currently only supported for PowerFlex.",
true,
ConfigKey.Scope.StoragePool,
null);
diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java
index b1fcf5c..b837e60 100644
--- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java
+++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java
@@ -31,6 +31,7 @@
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.api.SnapshotGroup;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
+import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
@@ -46,14 +47,12 @@
import com.cloud.server.ManagementServerImpl;
import com.cloud.storage.DiskOfferingVO;
import com.cloud.storage.Storage;
-import com.cloud.storage.StorageManager;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.component.ManagerBase;
-import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn;
@@ -140,7 +139,7 @@
long prev_chain_size = 0;
long virtual_size=0;
for (VolumeObjectTO volume : volumeTOs) {
- String volumeSnapshotName = String.format("%s-%s-%s-%s", ScaleIOUtil.VMSNAPSHOT_PREFIX, vmSnapshotVO.getId(),
+ String volumeSnapshotName = String.format("%s-%s-%s-%s-%s", ScaleIOUtil.VMSNAPSHOT_PREFIX, vmSnapshotVO.getId(), volume.getId(),
storagePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value());
srcVolumeDestSnapshotMap.put(volume.getPath(), volumeSnapshotName);
@@ -476,12 +475,6 @@
}
private ScaleIOGatewayClient getScaleIOClient(final Long storagePoolId) throws Exception {
- final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
- final String url = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_ENDPOINT).getValue();
- final String encryptedUsername = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_USERNAME).getValue();
- final String username = DBEncryptionUtil.decrypt(encryptedUsername);
- final String encryptedPassword = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD).getValue();
- final String password = DBEncryptionUtil.decrypt(encryptedPassword);
- return ScaleIOGatewayClient.getClient(url, username, password, false, clientTimeout);
+ return ScaleIOGatewayClientConnectionPool.getInstance().getClient(storagePoolId, storagePoolDetailsDao);
}
}
diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
index 7c4601e..9d35911 100644
--- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
+++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
@@ -1384,25 +1384,47 @@
if (storageCanCloneVolume && computeSupportsVolumeClone) {
s_logger.debug("Storage " + destDataStoreId + " can support cloning using a cached template and compute side is OK with volume cloning.");
- TemplateInfo templateOnPrimary = destPrimaryDataStore.getTemplate(srcTemplateInfo.getId());
+ GlobalLock lock = null;
+ TemplateInfo templateOnPrimary = null;
- if (templateOnPrimary == null) {
- templateOnPrimary = createManagedTemplateVolume(srcTemplateInfo, destPrimaryDataStore);
+ try {
+ String tmplIdManagedPoolIdLockString = "tmplId:" + srcTemplateInfo.getId() + "managedPoolId:" + destDataStoreId;
+ lock = GlobalLock.getInternLock(tmplIdManagedPoolIdLockString);
+ if (lock == null) {
+ throw new CloudRuntimeException("Unable to create managed storage template/volume, couldn't get global lock on " + tmplIdManagedPoolIdLockString);
+ }
+
+ int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600);
+ if (!lock.lock(storagePoolMaxWaitSeconds)) {
+ s_logger.debug("Unable to create managed storage template/volume, couldn't lock on " + tmplIdManagedPoolIdLockString);
+ throw new CloudRuntimeException("Unable to create managed storage template/volume, couldn't lock on " + tmplIdManagedPoolIdLockString);
+ }
+
+ templateOnPrimary = destPrimaryDataStore.getTemplate(srcTemplateInfo.getId());
if (templateOnPrimary == null) {
- throw new CloudRuntimeException("Failed to create template " + srcTemplateInfo.getUniqueName() + " on primary storage: " + destDataStoreId);
+ templateOnPrimary = createManagedTemplateVolume(srcTemplateInfo, destPrimaryDataStore);
+
+ if (templateOnPrimary == null) {
+ throw new CloudRuntimeException("Failed to create template " + srcTemplateInfo.getUniqueName() + " on primary storage: " + destDataStoreId);
+ }
}
- }
- // Copy the template to the template volume.
- VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId());
+ // Copy the template to the template volume.
+ VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId());
- if (templatePoolRef == null) {
- throw new CloudRuntimeException("Failed to find template " + srcTemplateInfo.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId());
- }
+ if (templatePoolRef == null) {
+ throw new CloudRuntimeException("Failed to find template " + srcTemplateInfo.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId());
+ }
- if (templatePoolRef.getDownloadState() == Status.NOT_DOWNLOADED) {
- copyTemplateToManagedTemplateVolume(srcTemplateInfo, templateOnPrimary, templatePoolRef, destPrimaryDataStore, destHost);
+ if (templatePoolRef.getDownloadState() == Status.NOT_DOWNLOADED) {
+ copyTemplateToManagedTemplateVolume(srcTemplateInfo, templateOnPrimary, templatePoolRef, destPrimaryDataStore, destHost);
+ }
+ } finally {
+ if (lock != null) {
+ lock.unlock();
+ lock.releaseRef();
+ }
}
if (destPrimaryDataStore.getPoolType() != StoragePoolType.PowerFlex) {
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
index 3a34cce..c785354 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
@@ -115,6 +115,7 @@
import com.cloud.storage.template.TemplateLocation;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
+import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.S3.S3Utils;
@@ -1769,8 +1770,14 @@
return new DirectDownloadAnswer(false, msg, true);
}
- s_logger.debug("Checking for free space on the host for downloading the template");
- if (!isEnoughSpaceForDownloadTemplateOnTemporaryLocation(cmd.getTemplateSize())) {
+ Long templateSize = null;
+ if (!org.apache.commons.lang.StringUtils.isBlank(cmd.getUrl())) {
+ String url = cmd.getUrl();
+ templateSize = UriUtils.getRemoteSize(url);
+ }
+
+ s_logger.debug("Checking for free space on the host for downloading the template with physical size: " + templateSize + " and virtual size: " + cmd.getTemplateSize());
+ if (!isEnoughSpaceForDownloadTemplateOnTemporaryLocation(templateSize)) {
String msg = "Not enough space on the defined temporary location to download the template " + cmd.getTemplateId();
s_logger.error(msg);
return new DirectDownloadAnswer(false, msg, true);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientConnectionPool.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientConnectionPool.java
new file mode 100644
index 0000000..2daf8e4
--- /dev/null
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientConnectionPool.java
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.datastore.client;
+
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
+import org.apache.log4j.Logger;
+
+import com.cloud.storage.StorageManager;
+import com.cloud.utils.crypt.DBEncryptionUtil;
+import com.google.common.base.Preconditions;
+
+public class ScaleIOGatewayClientConnectionPool {
+ private static final Logger LOGGER = Logger.getLogger(ScaleIOGatewayClientConnectionPool.class);
+
+ private ConcurrentHashMap<Long, ScaleIOGatewayClient> gatewayClients;
+
+ private static final ScaleIOGatewayClientConnectionPool instance;
+
+ static {
+ instance = new ScaleIOGatewayClientConnectionPool();
+ }
+
+ public static ScaleIOGatewayClientConnectionPool getInstance() {
+ return instance;
+ }
+
+ private ScaleIOGatewayClientConnectionPool() {
+ gatewayClients = new ConcurrentHashMap<Long, ScaleIOGatewayClient>();
+ }
+
+ public ScaleIOGatewayClient getClient(Long storagePoolId, StoragePoolDetailsDao storagePoolDetailsDao)
+ throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
+ Preconditions.checkArgument(storagePoolId != null && storagePoolId > 0, "Invalid storage pool id");
+
+ ScaleIOGatewayClient client = null;
+ synchronized (gatewayClients) {
+ client = gatewayClients.get(storagePoolId);
+ if (client == null) {
+ final String url = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_ENDPOINT).getValue();
+ final String encryptedUsername = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_USERNAME).getValue();
+ final String username = DBEncryptionUtil.decrypt(encryptedUsername);
+ final String encryptedPassword = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD).getValue();
+ final String password = DBEncryptionUtil.decrypt(encryptedPassword);
+ final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
+
+ client = new ScaleIOGatewayClientImpl(url, username, password, false, clientTimeout);
+ gatewayClients.put(storagePoolId, client);
+ LOGGER.debug("Added gateway client for the storage pool: " + storagePoolId);
+ }
+ }
+
+ return client;
+ }
+
+ public boolean removeClient(Long storagePoolId) {
+ Preconditions.checkArgument(storagePoolId != null && storagePoolId > 0, "Invalid storage pool id");
+
+ ScaleIOGatewayClient client = null;
+ synchronized (gatewayClients) {
+ client = gatewayClients.remove(storagePoolId);
+ }
+
+ if (client != null) {
+ LOGGER.debug("Removed gateway client for the storage pool: " + storagePoolId);
+ return true;
+ }
+
+ return false;
+ }
+}
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
index 5b93d4f..c981300 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
@@ -86,6 +86,15 @@
private String password;
private String sessionKey = null;
+ // The session token is valid for 8 hours from the time it was created, unless there has been no activity for 10 minutes
+ // Reference: https://cpsdocs.dellemc.com/bundle/PF_REST_API_RG/page/GUID-92430F19-9F44-42B6-B898-87D5307AE59B.html
+ private static final long MAX_VALID_SESSION_TIME_IN_MILLISECS = 8 * 60 * 60 * 1000; // 8 hrs
+ private static final long MAX_IDLE_TIME_IN_MILLISECS = 10 * 60 * 1000; // 10 mins
+ private static final long BUFFER_TIME_IN_MILLISECS = 30 * 1000; // keep 30 secs buffer before the expiration (to avoid any last-minute operations)
+
+ private long createTime = 0;
+ private long lastUsedTime = 0;
+
public ScaleIOGatewayClientImpl(final String url, final String username, final String password,
final boolean validateCertificate, final int timeout)
throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
@@ -117,14 +126,14 @@
this.username = username;
this.password = password;
- authenticate(username, password);
+ authenticate();
}
/////////////////////////////////////////////////////////////
//////////////// Private Helper Methods /////////////////////
/////////////////////////////////////////////////////////////
- private void authenticate(final String username, final String password) {
+ private void authenticate() {
final HttpGet request = new HttpGet(apiURI.toString() + "/login");
request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()));
try {
@@ -141,6 +150,24 @@
} catch (final IOException e) {
throw new CloudRuntimeException("Failed to authenticate PowerFlex API Gateway due to:" + e.getMessage());
}
+ long now = System.currentTimeMillis();
+ createTime = lastUsedTime = now;
+ }
+
+ private synchronized void renewClientSessionOnExpiry() {
+ if (isSessionExpired()) {
+ LOG.debug("Session expired, renewing");
+ authenticate();
+ }
+ }
+
+ private boolean isSessionExpired() {
+ long now = System.currentTimeMillis() + BUFFER_TIME_IN_MILLISECS;
+ if ((now - createTime) > MAX_VALID_SESSION_TIME_IN_MILLISECS ||
+ (now - lastUsedTime) > MAX_IDLE_TIME_IN_MILLISECS) {
+ return true;
+ }
+ return false;
}
private void checkAuthFailure(final HttpResponse response) {
@@ -176,9 +203,13 @@
}
private HttpResponse get(final String path) throws IOException {
+ renewClientSessionOnExpiry();
final HttpGet request = new HttpGet(apiURI.toString() + path);
request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((this.username + ":" + this.sessionKey).getBytes()));
final HttpResponse response = httpClient.execute(request);
+ synchronized (this) {
+ lastUsedTime = System.currentTimeMillis();
+ }
String responseStatus = (response != null) ? (response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase()) : "nil";
LOG.debug("GET request path: " + path + ", response: " + responseStatus);
checkAuthFailure(response);
@@ -186,6 +217,7 @@
}
private HttpResponse post(final String path, final Object obj) throws IOException {
+ renewClientSessionOnExpiry();
final HttpPost request = new HttpPost(apiURI.toString() + path);
request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((this.username + ":" + this.sessionKey).getBytes()));
request.setHeader("Content-type", "application/json");
@@ -200,6 +232,9 @@
}
}
final HttpResponse response = httpClient.execute(request);
+ synchronized (this) {
+ lastUsedTime = System.currentTimeMillis();
+ }
String responseStatus = (response != null) ? (response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase()) : "nil";
LOG.debug("POST request path: " + path + ", response: " + responseStatus);
checkAuthFailure(response);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
index de40535..c2300b9 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
@@ -44,6 +44,7 @@
import org.apache.cloudstack.storage.datastore.api.StoragePoolStatistics;
import org.apache.cloudstack.storage.datastore.api.VolumeStatistics;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
+import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
@@ -73,7 +74,6 @@
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeDetailsDao;
import com.cloud.utils.Pair;
-import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachineManager;
import com.google.common.base.Preconditions;
@@ -104,13 +104,7 @@
}
private ScaleIOGatewayClient getScaleIOClient(final Long storagePoolId) throws Exception {
- final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
- final String url = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_ENDPOINT).getValue();
- final String encryptedUsername = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_USERNAME).getValue();
- final String username = DBEncryptionUtil.decrypt(encryptedUsername);
- final String encryptedPassword = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD).getValue();
- final String password = DBEncryptionUtil.decrypt(encryptedPassword);
- return ScaleIOGatewayClient.getClient(url, username, password, false, clientTimeout);
+ return ScaleIOGatewayClientConnectionPool.getInstance().getClient(storagePoolId, storagePoolDetailsDao);
}
@Override
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
index 023d931..7995906 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
@@ -31,6 +31,8 @@
import javax.inject.Inject;
+import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
@@ -80,6 +82,8 @@
@Inject
private PrimaryDataStoreDao primaryDataStoreDao;
@Inject
+ private StoragePoolDetailsDao storagePoolDetailsDao;
+ @Inject
private StoragePoolHostDao storagePoolHostDao;
@Inject
private PrimaryDataStoreHelper dataStoreHelper;
@@ -232,14 +236,7 @@
List<String> connectedSdcIps = null;
try {
- Map <String, String> dataStoreDetails = primaryDataStoreDao.getDetails(dataStore.getId());
- final String url = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_ENDPOINT);
- final String encryptedUsername = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_USERNAME);
- final String username = DBEncryptionUtil.decrypt(encryptedUsername);
- final String encryptedPassword = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_PASSWORD);
- final String password = DBEncryptionUtil.decrypt(encryptedPassword);
- final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.value();
- ScaleIOGatewayClient client = ScaleIOGatewayClient.getClient(url, username, password, false, clientTimeout);
+ ScaleIOGatewayClient client = ScaleIOGatewayClientConnectionPool.getInstance().getClient(dataStore.getId(), storagePoolDetailsDao);
connectedSdcIps = client.listConnectedSdcIps();
} catch (NoSuchAlgorithmException | KeyManagementException | URISyntaxException e) {
LOGGER.error("Failed to create storage pool", e);
@@ -296,14 +293,7 @@
List<String> connectedSdcIps = null;
try {
- Map <String, String> dataStoreDetails = primaryDataStoreDao.getDetails(dataStore.getId());
- String url = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_ENDPOINT);
- String encryptedUsername = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_USERNAME);
- final String username = DBEncryptionUtil.decrypt(encryptedUsername);
- String encryptedPassword = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_PASSWORD);
- final String password = DBEncryptionUtil.decrypt(encryptedPassword);
- final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.value();
- ScaleIOGatewayClient client = ScaleIOGatewayClient.getClient(url, username, password, false, clientTimeout);
+ ScaleIOGatewayClient client = ScaleIOGatewayClientConnectionPool.getInstance().getClient(dataStore.getId(), storagePoolDetailsDao);
connectedSdcIps = client.listConnectedSdcIps();
} catch (NoSuchAlgorithmException | KeyManagementException | URISyntaxException e) {
LOGGER.error("Failed to create storage pool", e);
@@ -392,6 +382,8 @@
}
}
+ ScaleIOGatewayClientConnectionPool.getInstance().removeClient(dataStore.getId());
+
return dataStoreHelper.deletePrimaryDataStore(dataStore);
}
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
index e27f8bd..f672231 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
@@ -21,14 +21,15 @@
import java.net.URISyntaxException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
-import java.util.Map;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
+import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
@@ -39,11 +40,9 @@
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.dao.StoragePoolHostDao;
-import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.exception.CloudRuntimeException;
public class ScaleIOHostListener implements HypervisorHostListener {
@@ -55,6 +54,7 @@
@Inject private HostDao _hostDao;
@Inject private StoragePoolHostDao _storagePoolHostDao;
@Inject private PrimaryDataStoreDao _primaryDataStoreDao;
+ @Inject private StoragePoolDetailsDao _storagePoolDetailsDao;
@Override
public boolean hostAdded(long hostId) {
@@ -90,14 +90,7 @@
private boolean isHostSdcConnected(String hostIpAddress, long poolId) {
try {
- Map<String, String> dataStoreDetails = _primaryDataStoreDao.getDetails(poolId);
- final String url = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_ENDPOINT);
- final String encryptedUsername = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_USERNAME);
- final String username = DBEncryptionUtil.decrypt(encryptedUsername);
- final String encryptedPassword = dataStoreDetails.get(ScaleIOGatewayClient.GATEWAY_API_PASSWORD);
- final String password = DBEncryptionUtil.decrypt(encryptedPassword);
- final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(poolId);
- ScaleIOGatewayClient client = ScaleIOGatewayClient.getClient(url, username, password, false, clientTimeout);
+ ScaleIOGatewayClient client = ScaleIOGatewayClientConnectionPool.getInstance().getClient(poolId, _storagePoolDetailsDao);
return client.isSdcConnected(hostIpAddress);
} catch (NoSuchAlgorithmException | KeyManagementException | URISyntaxException e) {
s_logger.error("Failed to check host sdc connection", e);
diff --git a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycleTest.java b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycleTest.java
index c62371f..eed82ff 100644
--- a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycleTest.java
+++ b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycleTest.java
@@ -31,9 +31,7 @@
import static org.mockito.MockitoAnnotations.initMocks;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -44,8 +42,10 @@
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
+import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientImpl;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.provider.ScaleIOHostListener;
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
@@ -80,7 +80,6 @@
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.template.TemplateManager;
-import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.exception.CloudRuntimeException;
@PrepareForTest(ScaleIOGatewayClient.class)
@@ -90,6 +89,8 @@
@Mock
private PrimaryDataStoreDao primaryDataStoreDao;
@Mock
+ private StoragePoolDetailsDao storagePoolDetailsDao;
+ @Mock
private PrimaryDataStoreHelper dataStoreHelper;
@Mock
private ResourceManager resourceManager;
@@ -135,19 +136,9 @@
final DataStore dataStore = mock(DataStore.class);
when(dataStore.getId()).thenReturn(1L);
- Map <String, String> mockDataStoreDetails = new HashMap<>();
- mockDataStoreDetails.put(ScaleIOGatewayClient.GATEWAY_API_ENDPOINT, "https://192.168.1.19/api");
- String encryptedUsername = DBEncryptionUtil.encrypt("root");
- mockDataStoreDetails.put(ScaleIOGatewayClient.GATEWAY_API_USERNAME, encryptedUsername);
- String encryptedPassword = DBEncryptionUtil.encrypt("Password@123");
- mockDataStoreDetails.put(ScaleIOGatewayClient.GATEWAY_API_PASSWORD, encryptedPassword);
- when(primaryDataStoreDao.getDetails(1L)).thenReturn(mockDataStoreDetails);
-
PowerMockito.mockStatic(ScaleIOGatewayClient.class);
ScaleIOGatewayClientImpl client = mock(ScaleIOGatewayClientImpl.class);
- String username = DBEncryptionUtil.decrypt(encryptedUsername);
- String password = DBEncryptionUtil.decrypt(encryptedPassword);
- when(ScaleIOGatewayClient.getClient("https://192.168.1.19/api", username, password, false, 60)).thenReturn(client);
+ when(ScaleIOGatewayClientConnectionPool.getInstance().getClient(1L, storagePoolDetailsDao)).thenReturn(client);
List<String> connectedSdcIps = new ArrayList<>();
connectedSdcIps.add("192.168.1.1");
diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
index e219088..a81c39a 100644
--- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
+++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
@@ -3980,13 +3980,18 @@
throw new CloudRuntimeException("Router health checks are not enabled for router " + routerId);
}
- if (cmd.shouldPerformFreshChecks() && !routerService.performRouterHealthChecks(routerId)) {
- throw new CloudRuntimeException("Unable to perform fresh checks on router.");
+ if (cmd.shouldPerformFreshChecks()) {
+ Pair<Boolean, String> healthChecksresult = routerService.performRouterHealthChecks(routerId);
+ if (healthChecksresult == null) {
+ throw new CloudRuntimeException("Failed to initiate fresh checks on router.");
+ } else if (!healthChecksresult.first()) {
+ throw new CloudRuntimeException("Unable to perform fresh checks on router - " + healthChecksresult.second());
+ }
}
List<RouterHealthCheckResult> result = new ArrayList<>(routerHealthCheckResultDao.getHealthCheckResults(routerId));
if (result == null || result.size() == 0) {
- throw new CloudRuntimeException("Database had no entries for health checks for router. This could happen for " +
+ throw new CloudRuntimeException("No health check results found for the router. This could happen for " +
"a newly created router. Please wait for periodic results to populate or manually call for checks to execute.");
}
diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java
index 39d902f..29315c8 100644
--- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java
+++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java
@@ -71,6 +71,7 @@
import com.cloud.network.addr.PublicIp;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.RouterHealthCheckResultDao;
import com.cloud.network.dao.UserIpv6AddressDao;
import com.cloud.network.lb.LoadBalancingRule;
import com.cloud.network.router.VirtualRouter.RedundantState;
@@ -156,6 +157,8 @@
ConfigurationDao _configDao;
@Inject
VpcVirtualNetworkApplianceManager _vpcRouterMgr;
+ @Inject
+ RouterHealthCheckResultDao _routerHealthCheckResultDao;
protected final Map<HypervisorType, ConfigKey<String>> hypervisorsMap = new HashMap<>();
@@ -254,6 +257,7 @@
_accountMgr.checkAccess(caller, null, true, router);
_itMgr.expunge(router.getUuid());
+ _routerHealthCheckResultDao.expungeHealthChecks(router.getId());
_routerDao.remove(router.getId());
return router;
}
diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
index df0d4c1..0925048 100644
--- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
@@ -1268,68 +1268,68 @@
ex.printStackTrace();
}
}
+ }
- private List<String> getFailingChecks(DomainRouterVO router, GetRouterMonitorResultsAnswer answer) {
+ private List<String> getFailingChecks(DomainRouterVO router, GetRouterMonitorResultsAnswer answer) {
- if (answer == null) {
- s_logger.warn("Unable to fetch monitor results for router " + router);
- resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Communication failed");
- return Arrays.asList(CONNECTIVITY_TEST);
- } else if (!answer.getResult()) {
- s_logger.warn("Failed to fetch monitor results from router " + router + " with details: " + answer.getDetails());
- if (StringUtils.isNotBlank(answer.getDetails()) && answer.getDetails().equalsIgnoreCase(READONLY_FILESYSTEM_ERROR)) {
- resetRouterHealthChecksAndConnectivity(router.getId(), true, false, "Failed to write: " + answer.getDetails());
- return Arrays.asList(FILESYSTEM_WRITABLE_TEST);
- } else {
- resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Failed to fetch results with details: " + answer.getDetails());
- return Arrays.asList(CONNECTIVITY_TEST);
- }
+ if (answer == null) {
+ s_logger.warn("Unable to fetch monitor results for router " + router);
+ resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Communication failed");
+ return Arrays.asList(CONNECTIVITY_TEST);
+ } else if (!answer.getResult()) {
+ s_logger.warn("Failed to fetch monitor results from router " + router + " with details: " + answer.getDetails());
+ if (StringUtils.isNotBlank(answer.getDetails()) && answer.getDetails().equalsIgnoreCase(READONLY_FILESYSTEM_ERROR)) {
+ resetRouterHealthChecksAndConnectivity(router.getId(), true, false, "Failed to write: " + answer.getDetails());
+ return Arrays.asList(FILESYSTEM_WRITABLE_TEST);
} else {
- resetRouterHealthChecksAndConnectivity(router.getId(), true, true, "Successfully fetched data");
- updateDbHealthChecksFromRouterResponse(router.getId(), answer.getMonitoringResults());
- return answer.getFailingChecks();
+ resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Failed to fetch results with details: " + answer.getDetails());
+ return Arrays.asList(CONNECTIVITY_TEST);
+ }
+ } else {
+ resetRouterHealthChecksAndConnectivity(router.getId(), true, true, "Successfully fetched data");
+ updateDbHealthChecksFromRouterResponse(router.getId(), answer.getMonitoringResults());
+ return answer.getFailingChecks();
+ }
+ }
+
+ private void handleFailingChecks(DomainRouterVO router, List<String> failingChecks) {
+ if (failingChecks == null || failingChecks.size() == 0) {
+ return;
+ }
+
+ String alertMessage = "Health checks failed: " + failingChecks.size() + " failing checks on router " + router.getUuid();
+ _alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(),
+ alertMessage, alertMessage);
+ s_logger.warn(alertMessage + ". Checking failed health checks to see if router needs recreate");
+
+ String checkFailsToRecreateVr = RouterHealthChecksFailuresToRecreateVr.valueIn(router.getDataCenterId());
+ StringBuilder failingChecksEvent = new StringBuilder();
+ boolean recreateRouter = false;
+ for (int i = 0; i < failingChecks.size(); i++) {
+ String failedCheck = failingChecks.get(i);
+ if (i == 0) {
+ failingChecksEvent.append("Router ")
+ .append(router.getUuid())
+ .append(" has failing checks: ");
+ }
+
+ failingChecksEvent.append(failedCheck);
+ if (i < failingChecks.size() - 1) {
+ failingChecksEvent.append(", ");
+ }
+
+ if (StringUtils.isNotBlank(checkFailsToRecreateVr) && checkFailsToRecreateVr.contains(failedCheck)) {
+ recreateRouter = true;
}
}
- private void handleFailingChecks(DomainRouterVO router, List<String> failingChecks) {
- if (failingChecks == null || failingChecks.size() == 0) {
- return;
- }
+ ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,
+ Domain.ROOT_DOMAIN, EventTypes.EVENT_ROUTER_HEALTH_CHECKS, failingChecksEvent.toString());
- String alertMessage = "Health checks failed: " + failingChecks.size() + " failing checks on router " + router.getUuid();
- _alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(),
- alertMessage, alertMessage);
- s_logger.warn(alertMessage + ". Checking failed health checks to see if router needs recreate");
-
- String checkFailsToRecreateVr = RouterHealthChecksFailuresToRecreateVr.valueIn(router.getDataCenterId());
- StringBuilder failingChecksEvent = new StringBuilder();
- boolean recreateRouter = false;
- for (int i = 0; i < failingChecks.size(); i++) {
- String failedCheck = failingChecks.get(i);
- if (i == 0) {
- failingChecksEvent.append("Router ")
- .append(router.getUuid())
- .append(" has failing checks: ");
- }
-
- failingChecksEvent.append(failedCheck);
- if (i < failingChecks.size() - 1) {
- failingChecksEvent.append(", ");
- }
-
- if (StringUtils.isNotBlank(checkFailsToRecreateVr) && checkFailsToRecreateVr.contains(failedCheck)) {
- recreateRouter = true;
- }
- }
-
- ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,
- Domain.ROOT_DOMAIN, EventTypes.EVENT_ROUTER_HEALTH_CHECKS, failingChecksEvent.toString());
-
- if (recreateRouter) {
- s_logger.warn("Health Check Alert: Found failing checks in " +
- RouterHealthChecksFailuresToRecreateVrCK + ", attempting recreating router.");
- recreateRouter(router.getId());
- }
+ if (recreateRouter) {
+ s_logger.warn("Health Check Alert: Found failing checks in " +
+ RouterHealthChecksFailuresToRecreateVrCK + ", attempting recreating router.");
+ recreateRouter(router.getId());
}
}
@@ -1551,7 +1551,7 @@
String controlIP = getRouterControlIP(router);
if (StringUtils.isNotBlank(controlIP) && !controlIP.equals("0.0.0.0")) {
- final GetRouterMonitorResultsCommand command = new GetRouterMonitorResultsCommand(performFreshChecks);
+ final GetRouterMonitorResultsCommand command = new GetRouterMonitorResultsCommand(performFreshChecks, false);
command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlIP);
command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
try {
@@ -1576,8 +1576,40 @@
return null;
}
+ private GetRouterMonitorResultsAnswer performBasicTestsOnRouter(DomainRouterVO router) {
+ if (!RouterHealthChecksEnabled.value()) {
+ return null;
+ }
+
+ String controlIP = getRouterControlIP(router);
+ if (StringUtils.isNotBlank(controlIP) && !controlIP.equals("0.0.0.0")) {
+ final GetRouterMonitorResultsCommand command = new GetRouterMonitorResultsCommand(false, true);
+ command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlIP);
+ command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
+ try {
+ final Answer answer = _agentMgr.easySend(router.getHostId(), command);
+
+ if (answer == null) {
+ s_logger.warn("Unable to fetch basic router test results data from router " + router.getHostName());
+ return null;
+ }
+ if (answer instanceof GetRouterMonitorResultsAnswer) {
+ return (GetRouterMonitorResultsAnswer) answer;
+ } else {
+ s_logger.warn("Unable to fetch basic router test results from router " + router.getHostName() + " Received answer " + answer.getDetails());
+ return new GetRouterMonitorResultsAnswer(command, false, null, answer.getDetails());
+ }
+ } catch (final Exception e) {
+ s_logger.warn("Error while performing basic tests on router: " + router.getInstanceName(), e);
+ return null;
+ }
+ }
+
+ return null;
+ }
+
@Override
- public boolean performRouterHealthChecks(long routerId) {
+ public Pair<Boolean, String> performRouterHealthChecks(long routerId) {
DomainRouterVO router = _routerDao.findById(routerId);
if (router == null) {
@@ -1590,35 +1622,45 @@
s_logger.info("Running health check results for router " + router.getUuid());
- final GetRouterMonitorResultsAnswer answer;
+ GetRouterMonitorResultsAnswer answer = null;
+ String resultDetails = "";
boolean success = true;
- // Step 1: Update health check data on router and perform and retrieve health checks on router
- if (!updateRouterHealthChecksConfig(router)) {
- s_logger.warn("Unable to update health check config for fresh run successfully for router: " + router + ", so trying to fetch last result.");
- success = false;
- answer = fetchAndUpdateRouterHealthChecks(router, false);
- } else {
- s_logger.info("Successfully updated health check config for fresh run successfully for router: " + router);
- answer = fetchAndUpdateRouterHealthChecks(router, true);
- }
- // Step 2: Update health checks values in database. We do this irrespective of new health check config.
+ // Step 1: Perform basic tests to check the connectivity and file system on router
+ answer = performBasicTestsOnRouter(router);
if (answer == null) {
+ s_logger.debug("No results received for the basic tests on router: " + router);
+ resultDetails = "Basic tests results unavailable";
success = false;
- resetRouterHealthChecksAndConnectivity(routerId, false, false, "Communication failed");
} else if (!answer.getResult()) {
+ s_logger.debug("Basic tests failed on router: " + router);
+ resultDetails = "Basic tests failed - " + answer.getMonitoringResults();
success = false;
- if (StringUtils.isNotBlank(answer.getDetails()) && answer.getDetails().equalsIgnoreCase(READONLY_FILESYSTEM_ERROR)) {
- resetRouterHealthChecksAndConnectivity(routerId, true, false, "Failed to write: " + answer.getDetails());
- } else {
- resetRouterHealthChecksAndConnectivity(routerId, false, false, "Failed to fetch results with details: " + answer.getDetails());
- }
} else {
- resetRouterHealthChecksAndConnectivity(routerId, true, true, "Successfully fetched data");
- updateDbHealthChecksFromRouterResponse(routerId, answer.getMonitoringResults());
+ // Step 2: Update health check data on router and perform and retrieve health checks on router
+ if (!updateRouterHealthChecksConfig(router)) {
+ s_logger.warn("Unable to update health check config for fresh run successfully for router: " + router + ", so trying to fetch last result.");
+ success = false;
+ answer = fetchAndUpdateRouterHealthChecks(router, false);
+ } else {
+ s_logger.info("Successfully updated health check config for fresh run successfully for router: " + router);
+ answer = fetchAndUpdateRouterHealthChecks(router, true);
+ }
+
+ if (answer == null) {
+ resultDetails = "Failed to fetch and update health checks";
+ success = false;
+ } else if (!answer.getResult()) {
+ resultDetails = "Get health checks failed - " + answer.getMonitoringResults();
+ success = false;
+ }
}
- return success;
+ // Step 3: Update health checks values in database. We do this irrespective of new health check config.
+ List<String> failingChecks = getFailingChecks(router, answer);
+ handleFailingChecks(router, failingChecks);
+
+ return new Pair<Boolean, String>(success, resultDetails);
}
protected class UpdateRouterHealthChecksConfigTask extends ManagedContextRunnable {
@@ -1632,7 +1674,13 @@
s_logger.debug("Found " + routers.size() + " running routers. ");
for (final DomainRouterVO router : routers) {
- updateRouterHealthChecksConfig(router);
+ GetRouterMonitorResultsAnswer answer = performBasicTestsOnRouter(router);
+ if (answer != null && answer.getResult()) {
+ updateRouterHealthChecksConfig(router);
+ } else {
+ String resultDetails = (answer == null) ? "" : ", " + answer.getMonitoringResults();
+ s_logger.debug("Couldn't update health checks config on router: " + router + " as basic tests didn't succeed" + resultDetails);
+ }
}
} catch (final Exception ex) {
s_logger.error("Fail to complete the UpdateRouterHealthChecksConfigTask! ", ex);
@@ -1665,7 +1713,6 @@
return false;
}
- SetMonitorServiceCommand command = createMonitorServiceCommand(router, null,true, true);
String controlIP = getRouterControlIP(router);
if (StringUtils.isBlank(controlIP) || controlIP.equals("0.0.0.0")) {
s_logger.debug("Skipping update data on router " + router.getUuid() + " because controlIp is not correct.");
@@ -1675,6 +1722,7 @@
s_logger.info("Updating data for router health checks for router " + router.getUuid());
Answer origAnswer = null;
try {
+ SetMonitorServiceCommand command = createMonitorServiceCommand(router, null, true, true);
origAnswer = _agentMgr.easySend(router.getHostId(), command);
} catch (final Exception e) {
s_logger.error("Error while sending update data for health check to router: " + router.getInstanceName(), e);
diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
index 33ca12a..d1ddb52 100644
--- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
+++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
@@ -1292,7 +1292,7 @@
return volume;
} catch (Exception e) {
- throw new CloudRuntimeException("Exception caught during resize volume operation of volume UUID: " + volume.getUuid(), e);
+ throw new CloudRuntimeException("Couldn't resize volume: " + volume.getName() + ", " + e.getMessage(), e);
}
}
@@ -2676,6 +2676,10 @@
throw new InvalidParameterValueException("VolumeId: " + volumeId + " please attach this volume to a VM before create snapshot for it");
}
+ if (storagePool.getPoolType() == Storage.StoragePoolType.PowerFlex) {
+ throw new InvalidParameterValueException("Cannot perform this operation, unsupported on storage pool type " + storagePool.getPoolType());
+ }
+
return snapshotMgr.allocSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, snapshotName, null);
}
@@ -2706,7 +2710,13 @@
}
if (volume.getPoolId() == null) {
throw new InvalidParameterValueException("The volume doesn't belong to a storage pool so can't extract it");
+ } else {
+ StoragePoolVO poolVO = _storagePoolDao.findById(volume.getPoolId());
+ if (poolVO != null && poolVO.getPoolType() == Storage.StoragePoolType.PowerFlex) {
+ throw new InvalidParameterValueException("Cannot extract volume, this operation is unsupported for volumes on storage pool type " + poolVO.getPoolType());
+ }
}
+
// Extract activity only for detached volumes or for volumes whose
// instance is stopped
if (volume.getInstanceId() != null && ApiDBUtils.findVMInstanceById(volume.getInstanceId()).getState() != State.Stopped) {
diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
index 1479f8d..106709f 100644
--- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
+++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
@@ -4830,6 +4830,8 @@
destinationHost = _hostDao.findById(hostId);
if (destinationHost == null) {
throw new InvalidParameterValueException("Unable to find the host to deploy the VM, host id=" + hostId);
+ } else if (destinationHost.getResourceState() != ResourceState.Enabled || destinationHost.getStatus() != Status.Up ) {
+ throw new InvalidParameterValueException("Unable to deploy the VM as the host: " + destinationHost.getName() + " is not in the right state");
}
}
return destinationHost;
@@ -5097,8 +5099,6 @@
throw new InvalidParameterValueException("Unable to find service offering: " + serviceOfferingId);
}
- Long templateId = cmd.getTemplateId();
-
if (!serviceOffering.isDynamic()) {
for(String detail: cmd.getDetails().keySet()) {
if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) {
@@ -5107,6 +5107,8 @@
}
}
+ Long templateId = cmd.getTemplateId();
+
VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, templateId);
// Make sure a valid template ID was specified
if (template == null) {
@@ -5131,6 +5133,14 @@
}
}
+ Account caller = CallContext.current().getCallingAccount();
+ Long callerId = caller.getId();
+
+ boolean isRootAdmin = _accountService.isRootAdmin(callerId);
+
+ Long hostId = cmd.getHostId();
+ getDestinationHost(hostId, isRootAdmin);
+
String ipAddress = cmd.getIpAddress();
String ip6Address = cmd.getIp6Address();
String macAddress = cmd.getMacAddress();
@@ -5181,8 +5191,6 @@
}
// Add extraConfig to user_vm_details table
- Account caller = CallContext.current().getCallingAccount();
- Long callerId = caller.getId();
String extraConfig = cmd.getExtraConfig();
if (StringUtils.isNotBlank(extraConfig)) {
if (EnableAdditionalVmConfig.valueIn(callerId)) {
diff --git a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java
index 45bf4c1..abb1863 100644
--- a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java
+++ b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java
@@ -39,6 +39,7 @@
import com.cloud.network.vpc.PrivateGateway;
import com.cloud.user.Account;
import com.cloud.user.User;
+import com.cloud.utils.Pair;
import com.cloud.utils.component.ManagerBase;
import com.cloud.vm.DomainRouterVO;
import com.cloud.vm.VirtualMachineProfile;
@@ -249,8 +250,8 @@
}
@Override
- public boolean performRouterHealthChecks(long routerId) {
- return false;
+ public Pair<Boolean, String> performRouterHealthChecks(long routerId) {
+ return null;
}
@Override
