engine-storage: control download redirection
Add a global setting to control whether redirection is allowed while
downloading templates and volumes
Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
index 2e331ca..25f177e 100755
--- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
@@ -26,6 +26,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
+import java.util.List;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
@@ -81,6 +82,7 @@
private long maxTemplateSizeInBytes;
private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpMethodRetryHandler myretryhandler;
+ private boolean followRedirects = false;
public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
String user, String password, Proxy proxy, ResourceType resourceType) {
@@ -112,7 +114,7 @@
private GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
- request.setFollowRedirects(true);
+ request.setFollowRedirects(followRedirects);
return request;
}
@@ -336,6 +338,12 @@
} else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
status = Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
+ if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
+ && !followRedirects) {
+ errorString = String.format("Failed to download %s due to redirection, response code: %d",
+ downloadUrl, responseCode);
+ s_logger.error(errorString);
+ }
return true; //FIXME: retry?
}
return false;
@@ -537,4 +545,12 @@
return this;
}
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ if (this.request != null) {
+ this.request.setFollowRedirects(followRedirects);
+ }
+ }
}
diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
index dd452f2..a118a9ac 100644
--- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
@@ -60,7 +60,7 @@
protected GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
- request.setFollowRedirects(true);
+ request.setFollowRedirects(followRedirects);
if (!toFileSet) {
String[] parts = downloadUrl.split("/");
String filename = parts[parts.length - 1];
@@ -173,4 +173,12 @@
public void setStatus(Status status) {
this.status = status;
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ super.setFollowRedirects(followRedirects);
+ if (this.request != null) {
+ this.request.setFollowRedirects(followRedirects);
+ }
+ }
}
diff --git a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
index 44565c4..a259e79 100644
--- a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
@@ -34,6 +34,7 @@
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
@@ -44,6 +45,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
+import java.util.List;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static java.util.Arrays.asList;
@@ -72,8 +74,8 @@
private long downloadTime;
private long totalBytes;
private long maxTemplateSizeInByte;
-
private boolean resume = false;
+ private boolean followRedirects = false;
public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
@@ -91,7 +93,7 @@
this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
// Follow redirects
- this.getMethod.setFollowRedirects(true);
+ this.getMethod.setFollowRedirects(followRedirects);
// Set file extension.
this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
@@ -124,10 +126,11 @@
return 0;
}
- if (!HTTPUtils.verifyResponseCode(responseCode)) {
+ boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY,
+ HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects;
+ if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) {
errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
LOGGER.warn(errorString);
-
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
@@ -373,4 +376,12 @@
public String getFileExtension() {
return fileExtension;
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ if (this.getMethod != null) {
+ this.getMethod.setFollowRedirects(followRedirects);
+ }
+ }
}
diff --git a/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
index 5db3d24..9fb1ca4 100644
--- a/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
@@ -92,4 +92,6 @@
boolean isInited();
long getMaxTemplateSizeInBytes();
+
+ void setFollowRedirects(boolean followRedirects);
}
diff --git a/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java b/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
index f56e491..66058bb 100644
--- a/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
+++ b/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
@@ -43,6 +43,7 @@
protected long _start;
protected StorageLayer _storage;
protected boolean _inited = false;
+ protected boolean followRedirects = false;
private long maxTemplateSizeInBytes;
public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) {
@@ -149,4 +150,9 @@
public boolean isInited() {
return _inited;
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
index e8618d5..7ca5761 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
@@ -25,6 +25,7 @@
private String format;
private String url;
+ private boolean followRedirects;
public String getFormat() {
return format;
@@ -34,10 +35,15 @@
return url;
}
- public CheckUrlCommand(final String format,final String url) {
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public CheckUrlCommand(final String format, final String url, final boolean followRedirects) {
super();
this.format = format;
this.url = url;
+ this.followRedirects = followRedirects;
}
@Override
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
index 7e1ff0b..b6748dc 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
@@ -45,7 +45,11 @@
private Long templateSize;
private Storage.ImageFormat format;
- protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
+ private boolean followRedirects;
+
+ protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool,
+ final String checksum, final Map<String, String> headers, final Integer connectTimeout,
+ final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) {
this.url = url;
this.templateId = templateId;
this.destData = destData;
@@ -55,6 +59,7 @@
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
+ this.followRedirects = followRedirects;
}
public String getUrl() {
@@ -137,4 +142,12 @@
public int getWaitInMillSeconds() {
return getWait() * 1000;
}
+
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
index f131b3b..bdc00b3 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
@@ -24,8 +24,10 @@
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
- public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
- super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+ public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+ Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+ super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+ null, followRedirects);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
index dd88ad2..a7e16ea 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
@@ -25,7 +25,10 @@
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
- public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
- super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
+ public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+ Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout,
+ boolean followRedirects) {
+ super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+ connectionRequestTimeout, followRedirects);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
index a3edceb..7742994 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
@@ -24,8 +24,9 @@
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
- public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
- super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+ public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+ Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+ super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
index 0bf9c4d..0e51d78 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
@@ -24,8 +24,9 @@
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
- public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
- super(url, templateId, destPool, checksum, headers, null, null, null);
+ public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool,
+ final String checksum, final Map<String, String> headers) {
+ super(url, templateId, destPool, checksum, headers, null, null, null, false);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
index 80509b1..bd9a407 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
@@ -34,27 +34,30 @@
* Get direct template downloader from direct download command and destination pool
*/
public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
- String destPoolLocalPath,
- String temporaryDownloadPath) {
+ String destPoolLocalPath, String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
- return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
- cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+ return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
+ cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+ temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof HttpsDirectDownloadCommand) {
- return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
- cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
+ return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
+ cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+ cmd.getConnectionRequestTimeout(), temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof NfsDirectDownloadCommand) {
- return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
+ return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
+ cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
- return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
- cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+ return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
+ cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+ temporaryDownloadPath, cmd.isFollowRedirects());
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
- public static boolean checkUrlExistence(String url) {
+ public static boolean checkUrlExistence(String url, boolean followRedirects) {
try {
- DirectTemplateDownloader checker = getCheckerDownloader(url);
+ DirectTemplateDownloader checker = getCheckerDownloader(url, followRedirects);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -62,22 +65,22 @@
}
}
- private static DirectTemplateDownloader getCheckerDownloader(String url) {
+ private static DirectTemplateDownloader getCheckerDownloader(String url, boolean followRedirects) {
if (url.toLowerCase().startsWith("https:")) {
- return new HttpsDirectTemplateDownloader(url);
+ return new HttpsDirectTemplateDownloader(url, followRedirects);
} else if (url.toLowerCase().startsWith("http:")) {
- return new HttpDirectTemplateDownloader(url);
+ return new HttpDirectTemplateDownloader(url, followRedirects);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else if (url.toLowerCase().endsWith(".metalink")) {
- return new MetalinkDirectTemplateDownloader(url);
+ return new MetalinkDirectTemplateDownloader(url, followRedirects);
} else {
throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
}
}
- public static Long getFileSize(String url, String format) {
- DirectTemplateDownloader checker = getCheckerDownloader(url);
+ public static Long getFileSize(String url, String format, boolean followRedirects) {
+ DirectTemplateDownloader checker = getCheckerDownloader(url, followRedirects);
return checker.getRemoteFileSize(url, format);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
index 9476dba..9431b82 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
@@ -42,16 +42,19 @@
private String checksum;
private boolean redownload = false;
protected String temporaryDownloadPath;
+ private boolean followRedirects;
public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
- final String checksum, final String temporaryDownloadPath) {
+ final String checksum, final String temporaryDownloadPath,
+ final boolean followRedirects) {
this.url = url;
this.destPoolPath = destPoolPath;
this.templateId = templateId;
this.checksum = checksum;
this.temporaryDownloadPath = temporaryDownloadPath;
+ this.followRedirects = followRedirects;
}
private static String directDownloadDir = "template";
@@ -111,6 +114,14 @@
return redownload;
}
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
+
/**
* Create download directory (if it does not exist)
*/
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
index 093f060..577de9c 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
@@ -50,13 +50,14 @@
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
- protected HttpDirectTemplateDownloader(String url) {
- this(url, null, null, null, null, null, null, null);
+ protected HttpDirectTemplateDownloader(String url, boolean followRedirects) {
+ this(url, null, null, null, null, null, null, null, followRedirects);
}
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
- Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
- super(url, destPoolPath, templateId, checksum, downloadPath);
+ Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
+ boolean followRedirects) {
+ super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
client = new HttpClient(s_httpClientManager);
@@ -68,7 +69,7 @@
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
GetMethod request = new GetMethod(downloadUrl);
- request.setFollowRedirects(true);
+ request.setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String key : headers.keySet()) {
request.setRequestHeader(key, headers.get(key));
@@ -111,9 +112,11 @@
@Override
public boolean checkUrl(String url) {
HeadMethod httpHead = new HeadMethod(url);
+ httpHead.setFollowRedirects(this.isFollowRedirects());
try {
- if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
- s_logger.error(String.format("Invalid URL: %s", url));
+ int responseCode = client.executeMethod(httpHead);
+ if (responseCode != HttpStatus.SC_OK) {
+ s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;
@@ -128,9 +131,9 @@
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
- return QCOW2Utils.getVirtualSize(url);
+ return QCOW2Utils.getVirtualSizeFromUrl(url, this.isFollowRedirects());
} else {
- return UriUtils.getRemoteSize(url);
+ return UriUtils.getRemoteSize(url, this.isFollowRedirects());
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
index 2035aab..2e64e3c 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
@@ -19,29 +19,6 @@
package org.apache.cloudstack.direct.download;
-import com.cloud.utils.Pair;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.script.Script;
-import com.cloud.utils.storage.QCOW2Utils;
-import org.apache.cloudstack.utils.security.SSLUtils;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.io.IOUtils;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.commons.collections.MapUtils;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -60,25 +37,56 @@
import java.util.List;
import java.util.Map;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+
+import org.apache.cloudstack.utils.security.SSLUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+
+import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.QCOW2Utils;
+
public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected CloseableHttpClient httpsClient;
private HttpUriRequest req;
- protected HttpsDirectTemplateDownloader(String url) {
- this(url, null, null, null, null, null, null, null, null);
+ protected HttpsDirectTemplateDownloader(String url, boolean followRedirects) {
+ this(url, null, null, null, null, null, null, null, null, followRedirects);
}
- public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
- Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
- super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
+ public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
+ Map<String, String> headers, Integer connectTimeout, Integer soTimeout,
+ Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) {
+ super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects);
SSLContext sslcontext = getSSLContext();
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
- .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
- httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
+ .setSocketTimeout(soTimeout == null ? 5000 : soTimeout)
+ .setRedirectsEnabled(followRedirects)
+ .build();
+ httpsClient = HttpClients.custom()
+ .setSSLSocketFactory(factory)
+ .setDefaultRequestConfig(config)
+ .build();
createUriRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
File tempFile = createTemporaryDirectoryAndFile(downloadDir);
@@ -87,6 +95,7 @@
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
+ setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
@@ -161,8 +170,9 @@
HttpHead httpHead = new HttpHead(url);
try {
CloseableHttpResponse response = httpsClient.execute(httpHead);
- if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
- s_logger.error(String.format("Invalid URL: %s", url));
+ int responseCode = response.getStatusLine().getStatusCode();
+ if (responseCode != HttpStatus.SC_OK) {
+ s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
index 8380206..ef64d6e 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
@@ -42,16 +42,15 @@
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
- String destPoolPath, String checksum,
- Map<String, String> headers,
- Integer connectTimeout, Integer soTimeout,
- Integer connectionRequestTimeout, String temporaryDownloadPath) {
+ String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout,
+ Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
- connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
+ connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath,
+ this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
- connectTimeout, soTimeout, temporaryDownloadPath);
+ connectTimeout, soTimeout, temporaryDownloadPath, this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else {
@@ -60,13 +59,14 @@
}
}
- protected MetalinkDirectTemplateDownloader(String url) {
- this(url, null, null, null, null, null, null, null);
+ protected MetalinkDirectTemplateDownloader(String url, boolean followRedirects) {
+ this(url, null, null, null, null, null, null, null, followRedirects);
}
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
- Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
- super(url, destPoolPath, templateId, checksum, downloadPath);
+ Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
+ boolean followRedirects) {
+ super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
this.headers = headers;
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
index d606136..e5ff533 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
@@ -57,8 +57,9 @@
this(url, null, null, null, null);
}
- public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
- super(url, destPool, templateId, checksum, downloadPath);
+ public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum,
+ String downloadPath) {
+ super(url, destPool, templateId, checksum, downloadPath, false);
parseUrl();
}
diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
index 29d737f..7b41bd9 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
@@ -48,6 +48,8 @@
private DataStoreTO _store;
private DataStoreTO cacheStore;
+ private boolean followRedirects = false;
+
protected DownloadCommand() {
}
@@ -64,6 +66,7 @@
installPath = that.installPath;
_store = that._store;
_proxy = that._proxy;
+ followRedirects = that.followRedirects;
}
public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {
@@ -79,6 +82,7 @@
setSecUrl(((NfsTO)_store).getUrl());
}
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+ this.followRedirects = template.isFollowRedirects();
}
public DownloadCommand(TemplateObjectTO template, String user, String passwd, Long maxDownloadSizeInBytes) {
@@ -94,6 +98,7 @@
_store = volume.getDataStore();
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
resourceType = ResourceType.VOLUME;
+ this.followRedirects = volume.isFollowRedirects();
}
@Override
@@ -181,4 +186,12 @@
public DataStoreTO getCacheStore() {
return cacheStore;
}
+
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
}
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java
new file mode 100644
index 0000000..70db8fa
--- /dev/null
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.to;
+
+public class DownloadableObjectTO {
+ protected boolean followRedirects = false;
+
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
+}
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
index c62110b..72e6492 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
@@ -30,7 +30,7 @@
import com.cloud.agent.api.to.DataTO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
-public class SnapshotObjectTO implements DataTO {
+public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private VolumeObjectTO volume;
private String parentSnapshotPath;
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
index a405785..eafe8f8 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
@@ -28,7 +28,7 @@
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.template.VirtualMachineTemplate;
-public class TemplateObjectTO implements DataTO {
+public class TemplateObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private String origUrl;
private String uuid;
@@ -87,6 +87,7 @@
this.deployAsIs = template.isDeployAsIs();
this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
this.directDownload = template.isDirectDownload();
+ this.followRedirects = template.isFollowRedirects();
}
@Override
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
index 8473ea7..2bb67c8 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
@@ -33,7 +33,7 @@
import java.util.Arrays;
-public class VolumeObjectTO implements DataTO {
+public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
private String uuid;
private Volume.Type volumeType;
private DataStoreTO dataStore;
@@ -119,6 +119,7 @@
this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId();
this.passphrase = volume.getPassphrase();
this.encryptFormat = volume.getEncryptFormat();
+ this.followRedirects = volume.isFollowRedirects();
}
public String getUuid() {
diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
index e4f1d8f..3ab7501 100644
--- a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
+++ b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
@@ -56,7 +56,7 @@
private HttpEntity httpEntity;
@InjectMocks
- protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl);
+ protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, false);
@Before
public void init() throws IOException {
diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
index 9c6400b..bcc50a7 100644
--- a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
+++ b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
@@ -25,7 +25,7 @@
public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
@InjectMocks
- protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl);
+ protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, false);
@Test
public void testCheckUrlMetalink() {
metalinkDownloader.downloader = httpsDownloader;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java
new file mode 100644
index 0000000..63b0867d
--- /dev/null
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.engine.subsystem.api.storage;
+
+public interface DownloadableDataInfo extends DataObject {
+ default public boolean isFollowRedirects() {
+ return true;
+ }
+}
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
index 1d49e19..3bd3100 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
@@ -21,7 +21,7 @@
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.UserData;
-public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
+public interface TemplateInfo extends DownloadableDataInfo, VirtualMachineTemplate {
@Override
String getUniqueName();
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
index be16c20..19b852b 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
@@ -26,7 +26,7 @@
import com.cloud.storage.Volume;
import com.cloud.vm.VirtualMachine;
-public interface VolumeInfo extends DataObject, Volume {
+public interface VolumeInfo extends DownloadableDataInfo, Volume {
boolean isAttachedVM();
diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
index 28e7c89..7fdec90 100644
--- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
+++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
@@ -191,6 +191,10 @@
true,
ConfigKey.Scope.Global,
null);
+ static final ConfigKey<Boolean> DataStoreDownloadFollowRedirects = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
+ Boolean.class, "store.download.follow.redirects", "false",
+ "Whether HTTP redirect is followed during store downloads for objects such as template, volume etc.",
+ true, ConfigKey.Scope.Global);
/**
* should we execute in sequence not involving any storages?
diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index e6235a6..6c4fcab 100644
--- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -91,6 +91,7 @@
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@@ -366,6 +367,7 @@
toBeDownloaded.addAll(allTemplates);
final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
+ Boolean followRedirect = StorageManager.DataStoreDownloadFollowRedirects.value();
for (VMTemplateVO tmplt : allTemplates) {
String uniqueName = tmplt.getUniqueName();
TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
@@ -446,7 +448,8 @@
try {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
- tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
+ tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl(),
+ followRedirect));
} catch (ResourceAllocationException e) {
s_logger.warn(e.getMessage());
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, e.getMessage(), e.getMessage());
diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
index b688197..3883637 100644
--- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
+++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
@@ -23,6 +23,7 @@
import javax.inject.Inject;
+import com.cloud.storage.StorageManager;
import com.cloud.user.UserData;
import org.apache.log4j.Logger;
@@ -73,8 +74,10 @@
VMTemplatePoolDao templatePoolDao;
@Inject
TemplateDataStoreDao templateStoreDao;
+ final private boolean followRedirects;
public TemplateObject() {
+ this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(VMTemplateVO template, DataStore dataStore) {
@@ -573,4 +576,9 @@
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
}
diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
index 5ebee87..b0f426a 100644
--- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
+++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
@@ -23,6 +23,7 @@
import com.cloud.configuration.Resource.ResourceType;
import com.cloud.dc.VsphereStoragePolicyVO;
import com.cloud.dc.dao.VsphereStoragePolicyDao;
+import com.cloud.storage.StorageManager;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
@@ -117,6 +118,7 @@
private MigrationOptions migrationOptions;
private boolean directDownload;
private String vSphereStoragePolicyId;
+ private boolean followRedirects;
private final List<Volume.State> volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying,
Volume.State.Expunged);
@@ -127,6 +129,7 @@
public VolumeObject() {
_volStateMachine = Volume.State.getStateMachine();
+ this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(DataStore dataStore, VolumeVO volumeVO) {
@@ -930,4 +933,9 @@
public void setEncryptFormat(String encryptFormat) {
volumeVO.setEncryptFormat(encryptFormat);
}
+
+ @Override
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
}
diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
index 1ed37ab..b38b30e 100644
--- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
+++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
@@ -31,4 +31,5 @@
<T> void set(ConfigKey<T> key, T value);
<T> void createOrUpdateConfigObject(String componentName, ConfigKey<T> key, String value);
+ boolean isNewConfig(ConfigKey<?> configKey);
}
diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
index 75a3ea4..46a1de9 100644
--- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
+++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
@@ -81,6 +81,7 @@
List<Configurable> _configurables;
List<ScopedConfigStorage> _scopedStorages;
Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
+ Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
@@ -193,6 +194,7 @@
}
_configDao.persist(vo);
+ newConfigs.add(vo.getName());
} else {
boolean configUpdated = false;
if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) ||
@@ -343,4 +345,9 @@
return new Pair<>(groupId, subGroupId);
}
+
+ @Override
+ public boolean isNewConfig(ConfigKey<?> configKey) {
+ return newConfigs.contains(configKey.key());
+ }
}
diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
index fed784c..8dd6f71 100644
--- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
+++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
@@ -18,9 +18,14 @@
//
package org.apache.cloudstack.framework.config.impl;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
import org.apache.cloudstack.framework.config.ConfigKey;
import org.junit.Assert;
import org.junit.Test;
+import org.springframework.test.util.ReflectionTestUtils;
public class ConfigDepotImplTest {
@@ -40,4 +45,16 @@
}
}
+ @Test
+ public void testIsNewConfig() {
+ String validNewConfigKey = "CONFIG";
+ ConfigKey<Boolean> validNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG", "true", "", true);
+ ConfigKey<Boolean> invalidNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG1", "true", "", true);
+ Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
+ newConfigs.add(validNewConfigKey);
+ ReflectionTestUtils.setField(configDepotImpl, "newConfigs", newConfigs);
+ Assert.assertTrue(configDepotImpl.isNewConfig(validNewConfig));
+ Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig));
+ }
+
}
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
index 935bc8e..034c49f 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
@@ -37,9 +37,9 @@
final String url = cmd.getUrl();
s_logger.info("Checking URL: " + url);
Long remoteSize = null;
- boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
+ boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, cmd.isFollowRedirects());
if (checkResult) {
- remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
+ remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), cmd.isFollowRedirects());
if (remoteSize == null || remoteSize < 0) {
s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
"url %s, obtained size = %s", url, remoteSize));
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
index 69c28d4..6ebc93f 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
@@ -2329,7 +2329,7 @@
Long templateSize = null;
if (StringUtils.isNotBlank(cmd.getUrl())) {
String url = cmd.getUrl();
- templateSize = UriUtils.getRemoteSize(url);
+ templateSize = UriUtils.getRemoteSize(url, cmd.isFollowRedirects());
}
s_logger.debug("Checking for free space on the host for downloading the template with physical size: " + templateSize + " and virtual size: " + cmd.getTemplateSize());
diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
index 643f292..c6379a7 100644
--- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
@@ -89,6 +89,7 @@
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@@ -216,6 +217,7 @@
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.JoinBuilder;
@@ -347,6 +349,11 @@
@Inject
protected UserVmManager userVmManager;
+ @Inject
+ ConfigDepot configDepot;
+ @Inject
+ ConfigurationDao configurationDao;
+
protected List<StoragePoolDiscoverer> _discoverers;
public List<StoragePoolDiscoverer> getDiscoverers() {
@@ -407,6 +414,23 @@
}
}
+ protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() {
+ if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) {
+ if (s_logger.isTraceEnabled()) {
+ s_logger.trace(String.format("%s is not a new configuration, skipping updating its value",
+ DataStoreDownloadFollowRedirects.key()));
+ }
+ return;
+ }
+ List<DataCenterVO> zones =
+ _dcDao.listAll(new Filter(1));
+ if (CollectionUtils.isNotEmpty(zones)) {
+ s_logger.debug(String.format("Updating value for configuration: %s to true",
+ DataStoreDownloadFollowRedirects.key()));
+ configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true");
+ }
+ }
+
@Override
public List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type) {
List<StoragePoolVO> pools = _storagePoolDao.listByDataCenterId(datacenterId);
@@ -638,7 +662,7 @@
}
_executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
-
+ enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
return true;
}
@@ -3417,7 +3441,8 @@
SecStorageVMAutoScaleDown,
MountDisabledStoragePool,
VmwareCreateCloneFull,
- VmwareAllowParallelExecution
+ VmwareAllowParallelExecution,
+ DataStoreDownloadFollowRedirects
};
}
diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
index 69f455d..5a89191 100644
--- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
+++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
@@ -536,12 +536,14 @@
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
}
UriUtils.validateUrl(format, url);
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
s_logger.debug("Checking url: " + url);
- DirectDownloadHelper.checkUrlExistence(url);
+ DirectDownloadHelper.checkUrlExistence(url, followRedirects);
}
// Check that the resource limit for secondary storage won't be exceeded
- _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+ _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage,
+ UriUtils.getRemoteSize(url, followRedirects));
} else {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
}
@@ -643,7 +645,8 @@
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
//url can be null incase of postupload
if (url != null) {
- _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+ _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage,
+ UriUtils.getRemoteSize(url, StorageManager.DataStoreDownloadFollowRedirects.value()));
}
return volume;
diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
index f5c19ec..7216b27 100644
--- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
+++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
@@ -84,6 +84,7 @@
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
import com.cloud.storage.TemplateProfile;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
@@ -151,7 +152,8 @@
* Validate on random running KVM host that URL is reachable
* @param url url
*/
- private Long performDirectDownloadUrlValidation(final String format, final String url, final List<Long> zoneIds) {
+ private Long performDirectDownloadUrlValidation(final String format, final String url, final List<Long> zoneIds,
+ boolean followRedirects) {
HostVO host = null;
if (zoneIds != null && !zoneIds.isEmpty()) {
for (Long zoneId : zoneIds) {
@@ -167,7 +169,7 @@
if (host == null) {
throw new CloudRuntimeException("Couldn't find a host to validate URL " + url);
}
- CheckUrlCommand cmd = new CheckUrlCommand(format, url);
+ CheckUrlCommand cmd = new CheckUrlCommand(format, url, followRedirects);
s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
Answer answer = _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
@@ -191,6 +193,7 @@
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
List<Long> zoneIds = null;
@@ -198,12 +201,15 @@
zoneIds = new ArrayList<>();
zoneIds.add(cmd.getZoneId());
}
- Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(), url, zoneIds);
+ Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(), url, zoneIds,
+ followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
- _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+ _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+ ResourceType.secondary_storage,
+ UriUtils.getRemoteSize(url, followRedirects));
return profile;
}
@@ -221,14 +227,18 @@
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload());
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
- Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), url, cmd.getZoneIds());
+ Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), url, cmd.getZoneIds(),
+ followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
- _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+ _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+ ResourceType.secondary_storage,
+ UriUtils.getRemoteSize(url, followRedirects));
return profile;
}
diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
index 17cb969..3fc959d 100644
--- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
+++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
@@ -271,7 +271,8 @@
PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
DownloadProtocol protocol = getProtocolFromUrl(url);
- DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
+ DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum,
+ headers);
cmd.setTemplateSize(template.getSize());
cmd.setFormat(template.getFormat());
@@ -383,19 +384,23 @@
/**
* Return DirectDownloadCommand according to the protocol
*/
- private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
- String checksum, Map<String, String> httpHeaders) {
+ private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url,
+ Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> httpHeaders) {
int connectTimeout = DirectDownloadConnectTimeout.value();
int soTimeout = DirectDownloadSocketTimeout.value();
int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value();
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (protocol.equals(DownloadProtocol.HTTP)) {
- return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+ return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+ soTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.HTTPS)) {
- return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout);
+ return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+ soTimeout, connectionRequestTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.NFS)) {
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.METALINK)) {
- return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+ return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+ soTimeout, followRedirects);
} else {
return null;
}
diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
index e7004ba..970f0fa 100644
--- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
+++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
@@ -19,6 +19,8 @@
import java.util.ArrayList;
import java.util.List;
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -29,6 +31,8 @@
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.agent.api.StoragePoolInfo;
+import com.cloud.dc.DataCenterVO;
+import com.cloud.dc.dao.DataCenterDao;
import com.cloud.host.Host;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.vm.VMInstanceVO;
@@ -42,6 +46,12 @@
@Mock
VMInstanceDao vmInstanceDao;
+ @Mock
+ ConfigDepot configDepot;
+ @Mock
+ ConfigurationDao configurationDao;
+ @Mock
+ DataCenterDao dataCenterDao;
@Spy
@InjectMocks
@@ -128,4 +138,35 @@
Assert.assertTrue(storageManagerImpl.isVolumeSuspectedDestroyDuplicateOfVmVolume(volume));
}
+ @Test
+ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNoChange() {
+ Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+ .thenReturn(false);
+ storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+ Mockito.verify(configurationDao, Mockito.never()).update(Mockito.anyString(), Mockito.anyString());
+ }
+
+ @Test
+ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsOldInstall() {
+ Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+ .thenReturn(true);
+ Mockito.when(dataCenterDao.listAll(Mockito.any()))
+ .thenReturn(List.of(Mockito.mock(DataCenterVO.class)));
+ Mockito.doReturn(true).when(configurationDao).update(Mockito.anyString(), Mockito.anyString());
+ storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+ Mockito.verify(configurationDao, Mockito.times(1))
+ .update(StorageManager.DataStoreDownloadFollowRedirects.key(), "true");
+ }
+
+ @Test
+ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNewInstall() {
+ Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+ .thenReturn(true);
+ Mockito.when(dataCenterDao.listAll(Mockito.any()))
+ .thenReturn(new ArrayList<>()); //new installation
+ storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+ Mockito.verify(configurationDao, Mockito.never())
+ .update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue());
+ }
+
}
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
index a041770..5526835 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
@@ -41,17 +41,21 @@
* @param hvm whether the template is a hardware virtual machine
* @param accountId the accountId of the iso owner (null if public iso)
* @param descr description of the template
- * @param user username used for authentication to the server
- * @param password password used for authentication to the server
+ * @param userName username used for authentication to the server
+ * @param passwd password used for authentication to the server
* @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
* @param resourceType signifying the type of resource like template, volume etc.
+ * @param followRedirects whether downloader follows redirections
* @return job-id that can be used to interrogate the status of the download.
*/
- public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
+ public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String templatePath,
+ String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType,
+ boolean followRedirects);
- public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
+ public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+ long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects);
Map<String, Processor> getProcessors();
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index f647b49..e1497768 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -549,8 +549,9 @@
}
@Override
- public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+ public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+ long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
@@ -570,6 +571,7 @@
} else {
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
+ td.setFollowRedirects(followRedirects);
DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
dj.setTmpltPath(installPathPrefix);
jobs.put(jobId, dj);
@@ -579,8 +581,10 @@
}
@Override
- public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+ public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user,
+ String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType,
+ boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
String tmpDir = installPathPrefix;
@@ -632,6 +636,7 @@
} else {
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
+ td.setFollowRedirects(followRedirects);
// NOTE the difference between installPathPrefix and templatePath
// here. instalPathPrefix is the absolute path for template
// including mount directory
@@ -768,12 +773,16 @@
String jobId = null;
if (dstore instanceof S3TO) {
jobId =
- downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
- cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+ downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
+ cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
+ installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+ cmd.isFollowRedirects());
} else {
jobId =
- downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
- cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+ downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(),
+ cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix,
+ cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+ cmd.isFollowRedirects());
}
sleep();
if (jobId == null) {
diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java
index dffc010..13adfd7 100644
--- a/utils/src/main/java/com/cloud/utils/UriUtils.java
+++ b/utils/src/main/java/com/cloud/utils/UriUtils.java
@@ -213,7 +213,7 @@
}
// Get the size of a file from URL response header.
- public static long getRemoteSize(String url) {
+ public static long getRemoteSize(String url, Boolean followRedirect) {
long remoteSize = 0L;
final String[] methods = new String[]{"HEAD", "GET"};
IllegalArgumentException exception = null;
@@ -228,6 +228,7 @@
httpConn.setRequestMethod(method);
httpConn.setConnectTimeout(2000);
httpConn.setReadTimeout(5000);
+ httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));
String contentLength = httpConn.getHeaderField("content-length");
if (contentLength != null) {
remoteSize = Long.parseLong(contentLength);
diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
index 32a5472..300fc4d 100644
--- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
+++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
@@ -22,8 +22,9 @@
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -111,16 +112,23 @@
}
}
- public static long getVirtualSize(String urlStr) {
+ public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) {
+ HttpURLConnection httpConn = null;
try {
- URL url = new URL(urlStr);
- return getVirtualSizeFromInputStream(url.openStream());
- } catch (MalformedURLException e) {
+ URI url = new URI(urlStr);
+ httpConn = (HttpURLConnection)url.toURL().openConnection();
+ httpConn.setInstanceFollowRedirects(followRedirects);
+ return getVirtualSizeFromInputStream(httpConn.getInputStream());
+ } catch (URISyntaxException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr);
} catch (IOException e) {
LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
+ } finally {
+ if (httpConn != null) {
+ httpConn.disconnect();
+ }
}
}
}
