engine-storage: control download redirection
Add a global setting to control whether redirection is allowed while
downloading templates and volumes
core: some changes on SimpleHttpMultiFileDownloader
similar as HttpTemplateDownloader
Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
index d55c387..7ad8070 100755
--- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java
@@ -28,6 +28,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
+import java.util.List;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
@@ -80,6 +81,7 @@
private long maxTemplateSizeInBytes;
private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpMethodRetryHandler myretryhandler;
+ private boolean followRedirects = false;
public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
String user, String password, Proxy proxy, ResourceType resourceType) {
@@ -111,7 +113,7 @@
private GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
- request.setFollowRedirects(true);
+ request.setFollowRedirects(followRedirects);
return request;
}
@@ -337,6 +339,12 @@
} else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
status = Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
+ if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
+ && !followRedirects) {
+ errorString = String.format("Failed to download %s due to redirection, response code: %d",
+ downloadUrl, responseCode);
+ s_logger.error(errorString);
+ }
return true; //FIXME: retry?
}
return false;
@@ -538,4 +546,12 @@
return this;
}
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ if (this.request != null) {
+ this.request.setFollowRedirects(followRedirects);
+ }
+ }
}
diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
index dd452f2..a118a9ac 100644
--- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java
@@ -60,7 +60,7 @@
protected GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
- request.setFollowRedirects(true);
+ request.setFollowRedirects(followRedirects);
if (!toFileSet) {
String[] parts = downloadUrl.split("/");
String filename = parts[parts.length - 1];
@@ -173,4 +173,12 @@
public void setStatus(Status status) {
this.status = status;
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ super.setFollowRedirects(followRedirects);
+ if (this.request != null) {
+ this.request.setFollowRedirects(followRedirects);
+ }
+ }
}
diff --git a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
index 44565c4..a259e79 100644
--- a/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/S3TemplateDownloader.java
@@ -34,6 +34,7 @@
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
@@ -44,6 +45,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
+import java.util.List;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static java.util.Arrays.asList;
@@ -72,8 +74,8 @@
private long downloadTime;
private long totalBytes;
private long maxTemplateSizeInByte;
-
private boolean resume = false;
+ private boolean followRedirects = false;
public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
@@ -91,7 +93,7 @@
this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
// Follow redirects
- this.getMethod.setFollowRedirects(true);
+ this.getMethod.setFollowRedirects(followRedirects);
// Set file extension.
this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
@@ -124,10 +126,11 @@
return 0;
}
- if (!HTTPUtils.verifyResponseCode(responseCode)) {
+ boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY,
+ HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects;
+ if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) {
errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
LOGGER.warn(errorString);
-
status = Status.UNRECOVERABLE_ERROR;
return 0;
}
@@ -373,4 +376,12 @@
public String getFileExtension() {
return fileExtension;
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ if (this.getMethod != null) {
+ this.getMethod.setFollowRedirects(followRedirects);
+ }
+ }
}
diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java
index 7a0ce47..56cf76f 100644
--- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java
@@ -25,6 +25,7 @@
import java.io.RandomAccessFile;
import java.util.Date;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
@@ -73,6 +74,7 @@
private final HttpMethodRetryHandler retryHandler;
private HashMap<String, String> urlFileMap;
+ private boolean followRedirects = false;
public SimpleHttpMultiFileDownloader(StorageLayer storageLayer, String[] downloadUrls, String toDir,
DownloadCompleteCallback callback, long maxTemplateSizeInBytes,
@@ -94,7 +96,7 @@
private GetMethod createRequest(String downloadUrl) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
- request.setFollowRedirects(true);
+ request.setFollowRedirects(followRedirects);
return request;
}
@@ -170,7 +172,7 @@
urlFileMap.put(downloadUrl, currentToFile);
file = new File(currentToFile);
long localFileSize = checkLocalFileSizeForResume(resume, file);
- if (checkServerResponse(localFileSize)) return 0;
+ if (checkServerResponse(localFileSize, downloadUrl)) return 0;
if (!tryAndGetRemoteSize()) return 0;
if (!canHandleDownloadSize()) return 0;
checkAndSetDownloadSize();
@@ -317,7 +319,7 @@
return true;
}
- private boolean checkServerResponse(long localFileSize) throws IOException {
+ private boolean checkServerResponse(long localFileSize, String downloadUrl) throws IOException {
int responseCode = 0;
if (localFileSize > 0) {
@@ -331,6 +333,12 @@
} else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
currentStatus = Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
+ if (List.of(HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode)
+ && !followRedirects) {
+ errorString = String.format("Failed to download %s due to redirection, response code: %d",
+ downloadUrl, responseCode);
+ s_logger.error(errorString);
+ }
return true; //FIXME: retry?
}
return false;
@@ -478,4 +486,12 @@
public Map<String, String> getDownloadedFilesMap() {
return urlFileMap;
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ if (this.request != null) {
+ this.request.setFollowRedirects(followRedirects);
+ }
+ }
}
diff --git a/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
index 5db3d24..9fb1ca4 100644
--- a/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
+++ b/core/src/main/java/com/cloud/storage/template/TemplateDownloader.java
@@ -92,4 +92,6 @@
boolean isInited();
long getMaxTemplateSizeInBytes();
+
+ void setFollowRedirects(boolean followRedirects);
}
diff --git a/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java b/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
index f56e491..66058bb 100644
--- a/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
+++ b/core/src/main/java/com/cloud/storage/template/TemplateDownloaderBase.java
@@ -43,6 +43,7 @@
protected long _start;
protected StorageLayer _storage;
protected boolean _inited = false;
+ protected boolean followRedirects = false;
private long maxTemplateSizeInBytes;
public TemplateDownloaderBase(StorageLayer storage, String downloadUrl, String toDir, long maxTemplateSizeInBytes, DownloadCompleteCallback callback) {
@@ -149,4 +150,9 @@
public boolean isInited() {
return _inited;
}
+
+ @Override
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
index b1b76da..325f614 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
@@ -28,6 +28,7 @@
private Integer connectTimeout;
private Integer connectionRequestTimeout;
private Integer socketTimeout;
+ private boolean followRedirects;
public String getFormat() {
return format;
@@ -43,19 +44,25 @@
public Integer getSocketTimeout() { return socketTimeout; }
- public CheckUrlCommand(final String format,final String url) {
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public CheckUrlCommand(final String format, final String url, final boolean followRedirects) {
super();
this.format = format;
this.url = url;
+ this.followRedirects = followRedirects;
}
- public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+ public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, final boolean followRedirects) {
super();
this.format = format;
this.url = url;
this.connectTimeout = connectTimeout;
this.socketTimeout = socketTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
+ this.followRedirects = followRedirects;
}
@Override
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
index 7e1ff0b..b6748dc 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
@@ -45,7 +45,11 @@
private Long templateSize;
private Storage.ImageFormat format;
- protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
+ private boolean followRedirects;
+
+ protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool,
+ final String checksum, final Map<String, String> headers, final Integer connectTimeout,
+ final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) {
this.url = url;
this.templateId = templateId;
this.destData = destData;
@@ -55,6 +59,7 @@
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
this.connectionRequestTimeout = connectionRequestTimeout;
+ this.followRedirects = followRedirects;
}
public String getUrl() {
@@ -137,4 +142,12 @@
public int getWaitInMillSeconds() {
return getWait() * 1000;
}
+
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
index f131b3b..bdc00b3 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
@@ -24,8 +24,10 @@
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
- public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
- super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+ public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+ Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+ super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+ null, followRedirects);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
index dd88ad2..a7e16ea 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
@@ -25,7 +25,10 @@
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
- public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout) {
- super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, connectionRequestTimeout);
+ public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+ Map<String, String> headers, int connectTimeout, int soTimeout, int connectionRequestTimeout,
+ boolean followRedirects) {
+ super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout,
+ connectionRequestTimeout, followRedirects);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
index a3edceb..7742994 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
@@ -24,8 +24,9 @@
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
- public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers, int connectTimeout, int soTimeout) {
- super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null);
+ public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum,
+ Map<String, String> headers, int connectTimeout, int soTimeout, boolean followRedirects) {
+ super(url, templateId, destPool, checksum, headers, connectTimeout, soTimeout, null, followRedirects);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
index 0bf9c4d..0e51d78 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
@@ -24,8 +24,9 @@
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
- public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers) {
- super(url, templateId, destPool, checksum, headers, null, null, null);
+ public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool,
+ final String checksum, final Map<String, String> headers) {
+ super(url, templateId, destPool, checksum, headers, null, null, null, false);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
index 27e35b7..a00274e 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
@@ -34,27 +34,30 @@
* Get direct template downloader from direct download command and destination pool
*/
public static DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
- String destPoolLocalPath,
- String temporaryDownloadPath) {
+ String destPoolLocalPath, String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
- return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
- cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+ return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
+ cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+ temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof HttpsDirectDownloadCommand) {
- return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath, cmd.getChecksum(), cmd.getHeaders(),
- cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
+ return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPoolLocalPath,
+ cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+ cmd.getConnectionRequestTimeout(), temporaryDownloadPath, cmd.isFollowRedirects());
} else if (cmd instanceof NfsDirectDownloadCommand) {
- return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
+ return new NfsDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
+ cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
- return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
- cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
+ return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPoolLocalPath, cmd.getTemplateId(),
+ cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(),
+ temporaryDownloadPath, cmd.isFollowRedirects());
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
}
- public static boolean checkUrlExistence(String url) {
+ public static boolean checkUrlExistence(String url, boolean followRedirects) {
try {
- DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
+ DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -62,9 +65,9 @@
}
}
- public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+ public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
try {
- DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+ DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -72,27 +75,27 @@
}
}
- private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+ private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
if (url.toLowerCase().startsWith("https:")) {
- return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+ return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
} else if (url.toLowerCase().startsWith("http:")) {
- return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout);
+ return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else if (url.toLowerCase().endsWith(".metalink")) {
- return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout);
+ return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout, followRedirects);
} else {
throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
}
}
- public static Long getFileSize(String url, String format) {
- DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
+ public static Long getFileSize(String url, String format, boolean followRedirects) {
+ DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null, followRedirects);
return checker.getRemoteFileSize(url, format);
}
- public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
- DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+ public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
+ DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout, followRedirects);
return checker.getRemoteFileSize(url, format);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
index 9476dba..9431b82 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java
@@ -42,16 +42,19 @@
private String checksum;
private boolean redownload = false;
protected String temporaryDownloadPath;
+ private boolean followRedirects;
public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
- final String checksum, final String temporaryDownloadPath) {
+ final String checksum, final String temporaryDownloadPath,
+ final boolean followRedirects) {
this.url = url;
this.destPoolPath = destPoolPath;
this.templateId = templateId;
this.checksum = checksum;
this.temporaryDownloadPath = temporaryDownloadPath;
+ this.followRedirects = followRedirects;
}
private static String directDownloadDir = "template";
@@ -111,6 +114,14 @@
return redownload;
}
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
+
/**
* Create download directory (if it does not exist)
*/
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
index e1b2f1f..068f6b0 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
@@ -50,13 +50,14 @@
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
- protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
- this(url, null, null, null, null, connectTimeout, socketTimeout, null);
+ protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
+ this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
}
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
- Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
- super(url, destPoolPath, templateId, checksum, downloadPath);
+ Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
+ boolean followRedirects) {
+ super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
client = new HttpClient(s_httpClientManager);
@@ -68,7 +69,7 @@
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
GetMethod request = new GetMethod(downloadUrl);
- request.setFollowRedirects(true);
+ request.setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String key : headers.keySet()) {
request.setRequestHeader(key, headers.get(key));
@@ -111,9 +112,11 @@
@Override
public boolean checkUrl(String url) {
HeadMethod httpHead = new HeadMethod(url);
+ httpHead.setFollowRedirects(this.isFollowRedirects());
try {
- if (client.executeMethod(httpHead) != HttpStatus.SC_OK) {
- s_logger.error(String.format("Invalid URL: %s", url));
+ int responseCode = client.executeMethod(httpHead);
+ if (responseCode != HttpStatus.SC_OK) {
+ s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;
@@ -128,9 +131,9 @@
@Override
public Long getRemoteFileSize(String url, String format) {
if ("qcow2".equalsIgnoreCase(format)) {
- return QCOW2Utils.getVirtualSize(url);
+ return QCOW2Utils.getVirtualSizeFromUrl(url, this.isFollowRedirects());
} else {
- return UriUtils.getRemoteSize(url);
+ return UriUtils.getRemoteSize(url, this.isFollowRedirects());
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
index 70a3eb2..3a48ade 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
@@ -68,20 +68,26 @@
protected CloseableHttpClient httpsClient;
private HttpUriRequest req;
- protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
- this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null);
+ protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout, boolean followRedirects) {
+ this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null, followRedirects);
}
- public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
- Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
- super(url, destPoolPath, templateId, checksum, temporaryDownloadPath);
+ public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
+ Map<String, String> headers, Integer connectTimeout, Integer soTimeout,
+ Integer connectionRequestTimeout, String temporaryDownloadPath, boolean followRedirects) {
+ super(url, destPoolPath, templateId, checksum, temporaryDownloadPath, followRedirects);
SSLContext sslcontext = getSSLContext();
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectTimeout == null ? 5000 : connectTimeout)
.setConnectionRequestTimeout(connectionRequestTimeout == null ? 5000 : connectionRequestTimeout)
- .setSocketTimeout(soTimeout == null ? 5000 : soTimeout).build();
- httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build();
+ .setSocketTimeout(soTimeout == null ? 5000 : soTimeout)
+ .setRedirectsEnabled(followRedirects)
+ .build();
+ httpsClient = HttpClients.custom()
+ .setSSLSocketFactory(factory)
+ .setDefaultRequestConfig(config)
+ .build();
createUriRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
File tempFile = createTemporaryDirectoryAndFile(downloadDir);
@@ -90,6 +96,7 @@
protected void createUriRequest(String downloadUrl, Map<String, String> headers) {
req = new HttpGet(downloadUrl);
+ setFollowRedirects(this.isFollowRedirects());
if (MapUtils.isNotEmpty(headers)) {
for (String headerKey: headers.keySet()) {
req.setHeader(headerKey, headers.get(headerKey));
@@ -164,8 +171,9 @@
HttpHead httpHead = new HttpHead(url);
try {
CloseableHttpResponse response = httpsClient.execute(httpHead);
- if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
- s_logger.error(String.format("Invalid URL: %s", url));
+ int responseCode = response.getStatusLine().getStatusCode();
+ if (responseCode != HttpStatus.SC_OK) {
+ s_logger.error(String.format("HTTP HEAD request to URL: %s failed, response code: %d", url, responseCode));
return false;
}
return true;
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
index 06578d8..86b9788 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
@@ -42,16 +42,15 @@
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
protected DirectTemplateDownloader createDownloaderForMetalinks(String url, Long templateId,
- String destPoolPath, String checksum,
- Map<String, String> headers,
- Integer connectTimeout, Integer soTimeout,
- Integer connectionRequestTimeout, String temporaryDownloadPath) {
+ String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout,
+ Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
if (url.toLowerCase().startsWith("https:")) {
return new HttpsDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
- connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath);
+ connectTimeout, soTimeout, connectionRequestTimeout, temporaryDownloadPath,
+ this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("http:")) {
return new HttpDirectTemplateDownloader(url, templateId, destPoolPath, checksum, headers,
- connectTimeout, soTimeout, temporaryDownloadPath);
+ connectTimeout, soTimeout, temporaryDownloadPath, this.isFollowRedirects());
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else {
@@ -60,13 +59,14 @@
}
}
- protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
- this(url, null, null, null, null, connectTimeout, socketTimeout, null);
+ protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout, boolean followRedirects) {
+ this(url, null, null, null, null, connectTimeout, socketTimeout, null, followRedirects);
}
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
- Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
- super(url, destPoolPath, templateId, checksum, downloadPath);
+ Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath,
+ boolean followRedirects) {
+ super(url, destPoolPath, templateId, checksum, downloadPath, followRedirects);
this.headers = headers;
this.connectTimeout = connectTimeout;
this.soTimeout = soTimeout;
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
index d606136..e5ff533 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java
@@ -57,8 +57,9 @@
this(url, null, null, null, null);
}
- public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
- super(url, destPool, templateId, checksum, downloadPath);
+ public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum,
+ String downloadPath) {
+ super(url, destPool, templateId, checksum, downloadPath, false);
parseUrl();
}
diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
index 4032ac0..f44220f 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java
@@ -49,6 +49,8 @@
private DataStoreTO _store;
private DataStoreTO cacheStore;
+ private boolean followRedirects = false;
+
protected DownloadCommand() {
}
@@ -65,6 +67,7 @@
installPath = that.installPath;
_store = that._store;
_proxy = that._proxy;
+ followRedirects = that.followRedirects;
}
public DownloadCommand(TemplateObjectTO template, Long maxDownloadSizeInBytes) {
@@ -80,6 +83,7 @@
setSecUrl(((NfsTO)_store).getUrl());
}
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+ this.followRedirects = template.isFollowRedirects();
}
public DownloadCommand(TemplateObjectTO template, String user, String passwd, Long maxDownloadSizeInBytes) {
@@ -95,6 +99,7 @@
_store = volume.getDataStore();
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
resourceType = ResourceType.VOLUME;
+ this.followRedirects = volume.isFollowRedirects();
}
public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, String url) {
@@ -194,4 +199,12 @@
public DataStoreTO getCacheStore() {
return cacheStore;
}
+
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
}
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java
new file mode 100644
index 0000000..70db8fa
--- /dev/null
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/DownloadableObjectTO.java
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.to;
+
+public class DownloadableObjectTO {
+ protected boolean followRedirects = false;
+
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
+
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ }
+}
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
index 70cb6d1..76b9390 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java
@@ -30,7 +30,7 @@
import com.cloud.agent.api.to.DataTO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
-public class SnapshotObjectTO implements DataTO {
+public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private VolumeObjectTO volume;
private String parentSnapshotPath;
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
index a405785..eafe8f8 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
@@ -28,7 +28,7 @@
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.template.VirtualMachineTemplate;
-public class TemplateObjectTO implements DataTO {
+public class TemplateObjectTO extends DownloadableObjectTO implements DataTO {
private String path;
private String origUrl;
private String uuid;
@@ -87,6 +87,7 @@
this.deployAsIs = template.isDeployAsIs();
this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
this.directDownload = template.isDirectDownload();
+ this.followRedirects = template.isFollowRedirects();
}
@Override
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
index 8473ea7..2bb67c8 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
@@ -33,7 +33,7 @@
import java.util.Arrays;
-public class VolumeObjectTO implements DataTO {
+public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
private String uuid;
private Volume.Type volumeType;
private DataStoreTO dataStore;
@@ -119,6 +119,7 @@
this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId();
this.passphrase = volume.getPassphrase();
this.encryptFormat = volume.getEncryptFormat();
+ this.followRedirects = volume.isFollowRedirects();
}
public String getUuid() {
diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
index 2c72456..b74dc94 100644
--- a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
+++ b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
@@ -56,7 +56,7 @@
private HttpEntity httpEntity;
@InjectMocks
- protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000);
+ protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000, false);
@Before
public void init() throws IOException {
diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
index 68982fb..2fc0f7b 100644
--- a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
+++ b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
@@ -25,8 +25,7 @@
public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
@InjectMocks
- protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000);
-
+ protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000, false);
@Test
public void testCheckUrlMetalink() {
metalinkDownloader.downloader = httpsDownloader;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java
new file mode 100644
index 0000000..63b0867d
--- /dev/null
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DownloadableDataInfo.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.engine.subsystem.api.storage;
+
+public interface DownloadableDataInfo extends DataObject {
+ default public boolean isFollowRedirects() {
+ return true;
+ }
+}
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
index 1d49e19..3bd3100 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
@@ -21,7 +21,7 @@
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.UserData;
-public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
+public interface TemplateInfo extends DownloadableDataInfo, VirtualMachineTemplate {
@Override
String getUniqueName();
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
index be16c20..19b852b 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
@@ -26,7 +26,7 @@
import com.cloud.storage.Volume;
import com.cloud.vm.VirtualMachine;
-public interface VolumeInfo extends DataObject, Volume {
+public interface VolumeInfo extends DownloadableDataInfo, Volume {
boolean isAttachedVM();
diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
index f5cf443..b1594e3 100644
--- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
+++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
@@ -199,6 +199,10 @@
true,
ConfigKey.Scope.Global,
null);
+ static final ConfigKey<Boolean> DataStoreDownloadFollowRedirects = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
+ Boolean.class, "store.download.follow.redirects", "false",
+ "Whether HTTP redirect is followed during store downloads for objects such as template, volume etc.",
+ true, ConfigKey.Scope.Global);
ConfigKey<Long> HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000",
"The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true);
diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index e6235a6..6c4fcab 100644
--- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -91,6 +91,7 @@
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
@@ -366,6 +367,7 @@
toBeDownloaded.addAll(allTemplates);
final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
+ Boolean followRedirect = StorageManager.DataStoreDownloadFollowRedirects.value();
for (VMTemplateVO tmplt : allTemplates) {
String uniqueName = tmplt.getUniqueName();
TemplateDataStoreVO tmpltStore = _vmTemplateStoreDao.findByStoreTemplate(storeId, tmplt.getId());
@@ -446,7 +448,8 @@
try {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(accountId),
com.cloud.configuration.Resource.ResourceType.secondary_storage,
- tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl()));
+ tmpltInfo.getSize() - UriUtils.getRemoteSize(tmplt.getUrl(),
+ followRedirect));
} catch (ResourceAllocationException e) {
s_logger.warn(e.getMessage());
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, zoneId, null, e.getMessage(), e.getMessage());
diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
index b688197..3883637 100644
--- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
+++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java
@@ -23,6 +23,7 @@
import javax.inject.Inject;
+import com.cloud.storage.StorageManager;
import com.cloud.user.UserData;
import org.apache.log4j.Logger;
@@ -73,8 +74,10 @@
VMTemplatePoolDao templatePoolDao;
@Inject
TemplateDataStoreDao templateStoreDao;
+ final private boolean followRedirects;
public TemplateObject() {
+ this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(VMTemplateVO template, DataStore dataStore) {
@@ -573,4 +576,9 @@
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
}
diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
index 9e41e0d..2e49698 100644
--- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
+++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
@@ -23,6 +23,7 @@
import com.cloud.configuration.Resource.ResourceType;
import com.cloud.dc.VsphereStoragePolicyVO;
import com.cloud.dc.dao.VsphereStoragePolicyDao;
+import com.cloud.storage.StorageManager;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
@@ -117,6 +118,7 @@
private MigrationOptions migrationOptions;
private boolean directDownload;
private String vSphereStoragePolicyId;
+ private boolean followRedirects;
private final List<Volume.State> volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying,
Volume.State.Expunged);
@@ -127,6 +129,7 @@
public VolumeObject() {
_volStateMachine = Volume.State.getStateMachine();
+ this.followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
}
protected void configure(DataStore dataStore, VolumeVO volumeVO) {
@@ -930,4 +933,9 @@
public void setEncryptFormat(String encryptFormat) {
volumeVO.setEncryptFormat(encryptFormat);
}
+
+ @Override
+ public boolean isFollowRedirects() {
+ return followRedirects;
+ }
}
diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
index 1ed37ab..b38b30e 100644
--- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
+++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java
@@ -31,4 +31,5 @@
<T> void set(ConfigKey<T> key, T value);
<T> void createOrUpdateConfigObject(String componentName, ConfigKey<T> key, String value);
+ boolean isNewConfig(ConfigKey<?> configKey);
}
diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
index 75a3ea4..46a1de9 100644
--- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
+++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java
@@ -81,6 +81,7 @@
List<Configurable> _configurables;
List<ScopedConfigStorage> _scopedStorages;
Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
+ Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
@@ -193,6 +194,7 @@
}
_configDao.persist(vo);
+ newConfigs.add(vo.getName());
} else {
boolean configUpdated = false;
if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) ||
@@ -343,4 +345,9 @@
return new Pair<>(groupId, subGroupId);
}
+
+ @Override
+ public boolean isNewConfig(ConfigKey<?> configKey) {
+ return newConfigs.contains(configKey.key());
+ }
}
diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
index fed784c..8dd6f71 100644
--- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
+++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java
@@ -18,9 +18,14 @@
//
package org.apache.cloudstack.framework.config.impl;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
import org.apache.cloudstack.framework.config.ConfigKey;
import org.junit.Assert;
import org.junit.Test;
+import org.springframework.test.util.ReflectionTestUtils;
public class ConfigDepotImplTest {
@@ -40,4 +45,16 @@
}
}
+ @Test
+ public void testIsNewConfig() {
+ String validNewConfigKey = "CONFIG";
+ ConfigKey<Boolean> validNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG", "true", "", true);
+ ConfigKey<Boolean> invalidNewConfig = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, "CONFIG1", "true", "", true);
+ Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
+ newConfigs.add(validNewConfigKey);
+ ReflectionTestUtils.setField(configDepotImpl, "newConfigs", newConfigs);
+ Assert.assertTrue(configDepotImpl.isNewConfig(validNewConfig));
+ Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig));
+ }
+
}
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
index 5faad56..0e1bf57 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
@@ -42,9 +42,9 @@
s_logger.info(String.format("Checking URL: %s, with connect timeout: %d, connect request timeout: %d, socket timeout: %d", url, connectTimeout, connectionRequestTimeout, socketTimeout));
Long remoteSize = null;
- boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+ boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
if (checkResult) {
- remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout);
+ remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout, cmd.isFollowRedirects());
if (remoteSize == null || remoteSize < 0) {
s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
"url %s, obtained size = %s", url, remoteSize));
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
index 1be4a8b..f0ce56e 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
@@ -2378,7 +2378,7 @@
Long templateSize = null;
if (StringUtils.isNotBlank(cmd.getUrl())) {
String url = cmd.getUrl();
- templateSize = UriUtils.getRemoteSize(url);
+ templateSize = UriUtils.getRemoteSize(url, cmd.isFollowRedirects());
}
s_logger.debug("Checking for free space on the host for downloading the template with physical size: " + templateSize + " and virtual size: " + cmd.getTemplateSize());
diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
index 04889d0..85e7a4c 100644
--- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
@@ -97,6 +97,7 @@
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
+import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@@ -239,6 +240,7 @@
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.JoinBuilder;
@@ -382,6 +384,11 @@
@Inject
protected BucketDao _bucketDao;
+ @Inject
+ ConfigDepot configDepot;
+ @Inject
+ ConfigurationDao configurationDao;
+
protected List<StoragePoolDiscoverer> _discoverers;
public List<StoragePoolDiscoverer> getDiscoverers() {
@@ -442,6 +449,23 @@
}
}
+ protected void enableDefaultDatastoreDownloadRedirectionForExistingInstallations() {
+ if (!configDepot.isNewConfig(DataStoreDownloadFollowRedirects)) {
+ if (s_logger.isTraceEnabled()) {
+ s_logger.trace(String.format("%s is not a new configuration, skipping updating its value",
+ DataStoreDownloadFollowRedirects.key()));
+ }
+ return;
+ }
+ List<DataCenterVO> zones =
+ _dcDao.listAll(new Filter(1));
+ if (CollectionUtils.isNotEmpty(zones)) {
+ s_logger.debug(String.format("Updating value for configuration: %s to true",
+ DataStoreDownloadFollowRedirects.key()));
+ configurationDao.update(DataStoreDownloadFollowRedirects.key(), "true");
+ }
+ }
+
@Override
public List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type) {
List<StoragePoolVO> pools = _storagePoolDao.listByDataCenterId(datacenterId);
@@ -673,7 +697,7 @@
}
_executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
-
+ enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
return true;
}
@@ -3717,7 +3741,8 @@
MountDisabledStoragePool,
VmwareCreateCloneFull,
VmwareAllowParallelExecution,
- ConvertVmwareInstanceToKvmTimeout
+ ConvertVmwareInstanceToKvmTimeout,
+ DataStoreDownloadFollowRedirects
};
}
diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
index 2a0821c..a213a55 100644
--- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
+++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
@@ -547,12 +547,14 @@
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
}
UriUtils.validateUrl(format, url);
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (VolumeUrlCheck.value()) { // global setting that can be set when their MS does not have internet access
s_logger.debug("Checking url: " + url);
- DirectDownloadHelper.checkUrlExistence(url);
+ DirectDownloadHelper.checkUrlExistence(url, followRedirects);
}
// Check that the resource limit for secondary storage won't be exceeded
- _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+ _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage,
+ UriUtils.getRemoteSize(url, followRedirects));
} else {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
}
@@ -654,8 +656,10 @@
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
//url can be null incase of postupload
if (url != null) {
- _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
- volume.setSize(UriUtils.getRemoteSize(url));
+ long remoteSize = UriUtils.getRemoteSize(url, StorageManager.DataStoreDownloadFollowRedirects.value());
+ _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage,
+ remoteSize);
+ volume.setSize(remoteSize);
}
return volume;
diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
index b886f08..d8132df 100644
--- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
+++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
@@ -89,6 +89,7 @@
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
+import com.cloud.storage.StorageManager;
import com.cloud.storage.TemplateProfile;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
@@ -161,7 +162,7 @@
* @param url url
*/
private Long performDirectDownloadUrlValidation(final String format, final Hypervisor.HypervisorType hypervisor,
- final String url, final List<Long> zoneIds) {
+ final String url, final List<Long> zoneIds, final boolean followRedirects) {
HostVO host = null;
if (zoneIds != null && !zoneIds.isEmpty()) {
for (Long zoneId : zoneIds) {
@@ -180,7 +181,7 @@
Integer socketTimeout = DirectDownloadManager.DirectDownloadSocketTimeout.value();
Integer connectRequestTimeout = DirectDownloadManager.DirectDownloadConnectionRequestTimeout.value();
Integer connectTimeout = DirectDownloadManager.DirectDownloadConnectTimeout.value();
- CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout);
+ CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout, followRedirects);
s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
Answer answer = _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
@@ -204,6 +205,7 @@
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
List<Long> zoneIds = null;
@@ -212,12 +214,14 @@
zoneIds.add(cmd.getZoneId());
}
Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(),
- Hypervisor.HypervisorType.KVM, url, zoneIds);
+ Hypervisor.HypervisorType.KVM, url, zoneIds, followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
- _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+ _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+ ResourceType.secondary_storage,
+ UriUtils.getRemoteSize(url, followRedirects));
return profile;
}
@@ -236,15 +240,18 @@
String url = profile.getUrl();
UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload());
Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.getType(cmd.getHypervisor());
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (cmd.isDirectDownload()) {
DigestHelper.validateChecksumString(cmd.getChecksum());
Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(),
- hypervisor, url, cmd.getZoneIds());
+ hypervisor, url, cmd.getZoneIds(), followRedirects);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
- _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
+ _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()),
+ ResourceType.secondary_storage,
+ UriUtils.getRemoteSize(url, followRedirects));
return profile;
}
diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
index af543c6..0a21815 100644
--- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
+++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
@@ -272,7 +272,8 @@
PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
DownloadProtocol protocol = getProtocolFromUrl(url);
- DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
+ DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum,
+ headers);
cmd.setTemplateSize(template.getSize());
cmd.setFormat(template.getFormat());
@@ -393,19 +394,23 @@
/**
* Return DirectDownloadCommand according to the protocol
*/
- private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
- String checksum, Map<String, String> httpHeaders) {
+ private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url,
+ Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> httpHeaders) {
int connectTimeout = DirectDownloadConnectTimeout.value();
int soTimeout = DirectDownloadSocketTimeout.value();
int connectionRequestTimeout = DirectDownloadConnectionRequestTimeout.value();
+ boolean followRedirects = StorageManager.DataStoreDownloadFollowRedirects.value();
if (protocol.equals(DownloadProtocol.HTTP)) {
- return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+ return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+ soTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.HTTPS)) {
- return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout, connectionRequestTimeout);
+ return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+ soTimeout, connectionRequestTimeout, followRedirects);
} else if (protocol.equals(DownloadProtocol.NFS)) {
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.METALINK)) {
- return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout, soTimeout);
+ return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders, connectTimeout,
+ soTimeout, followRedirects);
} else {
return null;
}
diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
index ba5f2ba..dbceac9 100644
--- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
+++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java
@@ -17,12 +17,17 @@
package com.cloud.storage;
import com.cloud.agent.api.StoragePoolInfo;
+import com.cloud.dc.DataCenterVO;
+import com.cloud.dc.dao.DataCenterDao;
import com.cloud.exception.ConnectionException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.host.Host;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.dao.VMInstanceDao;
+
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.commons.collections.MapUtils;
@@ -48,6 +53,12 @@
@Mock
VMInstanceDao vmInstanceDao;
+ @Mock
+ ConfigDepot configDepot;
+ @Mock
+ ConfigurationDao configurationDao;
+ @Mock
+ DataCenterDao dataCenterDao;
@Spy
@InjectMocks
@@ -206,4 +217,36 @@
throw new RuntimeException(e);
}
}
+
+ @Test
+ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNoChange() {
+ Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+ .thenReturn(false);
+ storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+ Mockito.verify(configurationDao, Mockito.never()).update(Mockito.anyString(), Mockito.anyString());
+ }
+
+ @Test
+ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsOldInstall() {
+ Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+ .thenReturn(true);
+ Mockito.when(dataCenterDao.listAll(Mockito.any()))
+ .thenReturn(List.of(Mockito.mock(DataCenterVO.class)));
+ Mockito.doReturn(true).when(configurationDao).update(Mockito.anyString(), Mockito.anyString());
+ storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+ Mockito.verify(configurationDao, Mockito.times(1))
+ .update(StorageManager.DataStoreDownloadFollowRedirects.key(), "true");
+ }
+
+ @Test
+ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallationsNewInstall() {
+ Mockito.when(configDepot.isNewConfig(StorageManager.DataStoreDownloadFollowRedirects))
+ .thenReturn(true);
+ Mockito.when(dataCenterDao.listAll(Mockito.any()))
+ .thenReturn(new ArrayList<>()); //new installation
+ storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations();
+ Mockito.verify(configurationDao, Mockito.never())
+ .update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue());
+ }
+
}
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
index a041770..5526835 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManager.java
@@ -41,17 +41,21 @@
* @param hvm whether the template is a hardware virtual machine
* @param accountId the accountId of the iso owner (null if public iso)
* @param descr description of the template
- * @param user username used for authentication to the server
- * @param password password used for authentication to the server
+ * @param userName username used for authentication to the server
+ * @param passwd password used for authentication to the server
* @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
* @param resourceType signifying the type of resource like template, volume etc.
+ * @param followRedirects whether downloader follows redirections
* @return job-id that can be used to interrogate the status of the download.
*/
- public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
+ public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String templatePath,
+ String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType,
+ boolean followRedirects);
- public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
+ public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+ long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects);
Map<String, Processor> getProcessors();
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index 0396f96..e1e6902 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -661,8 +661,9 @@
}
@Override
- public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+ public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String user, String password,
+ long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType, boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
@@ -682,6 +683,7 @@
} else {
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
+ td.setFollowRedirects(followRedirects);
DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
dj.setTmpltPath(installPathPrefix);
jobs.put(jobId, dj);
@@ -717,8 +719,10 @@
}
@Override
- public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
- String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+ public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm,
+ Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user,
+ String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType,
+ boolean followRedirects) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
String tmpDir = installPathPrefix;
@@ -765,6 +769,7 @@
throw new CloudRuntimeException("Unable to download from URL: " + url);
}
}
+ td.setFollowRedirects(followRedirects);
// NOTE the difference between installPathPrefix and templatePath
// here. instalPathPrefix is the absolute path for template
// including mount directory
@@ -901,12 +906,16 @@
String jobId = null;
if (dstore instanceof S3TO) {
jobId =
- downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
- cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+ downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(),
+ cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
+ installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+ cmd.isFollowRedirects());
} else {
jobId =
- downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
- cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+ downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(),
+ cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix,
+ cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType,
+ cmd.isFollowRedirects());
}
sleep();
if (jobId == null) {
diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java
index a2bfa9e..7327218 100644
--- a/utils/src/main/java/com/cloud/utils/UriUtils.java
+++ b/utils/src/main/java/com/cloud/utils/UriUtils.java
@@ -213,7 +213,7 @@
}
// Get the size of a file from URL response header.
- public static long getRemoteSize(String url) {
+ public static long getRemoteSize(String url, Boolean followRedirect) {
long remoteSize = 0L;
final String[] methods = new String[]{"HEAD", "GET"};
IllegalArgumentException exception = null;
@@ -228,6 +228,7 @@
httpConn.setRequestMethod(method);
httpConn.setConnectTimeout(2000);
httpConn.setReadTimeout(5000);
+ httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));
String contentLength = httpConn.getHeaderField("content-length");
if (contentLength != null) {
remoteSize = Long.parseLong(contentLength);
diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
index 4daf138..a36b011 100644
--- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
+++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
@@ -22,8 +22,9 @@
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -112,16 +113,23 @@
}
}
- public static long getVirtualSize(String urlStr) {
+ public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects) {
+ HttpURLConnection httpConn = null;
try {
- URL url = new URL(urlStr);
- return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
- } catch (MalformedURLException e) {
+ URI url = new URI(urlStr);
+ httpConn = (HttpURLConnection)url.toURL().openConnection();
+ httpConn.setInstanceFollowRedirects(followRedirects);
+ return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr));
+ } catch (URISyntaxException e) {
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
throw new IllegalArgumentException("Invalid URL: " + urlStr);
} catch (IOException e) {
LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
+ } finally {
+ if (httpConn != null) {
+ httpConn.disconnect();
+ }
}
}
}
