| // |
| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| // |
| |
| package com.cloud.storage.template; |
| |
| import com.amazonaws.event.ProgressEvent; |
| import com.amazonaws.event.ProgressEventType; |
| import com.amazonaws.event.ProgressListener; |
| import com.amazonaws.services.s3.model.ObjectMetadata; |
| import com.amazonaws.services.s3.model.PutObjectRequest; |
| import com.amazonaws.services.s3.model.StorageClass; |
| import com.amazonaws.services.s3.transfer.Upload; |
| import com.cloud.agent.api.to.S3TO; |
| import com.cloud.utils.net.HTTPUtils; |
| import com.cloud.utils.net.Proxy; |
| import com.cloud.utils.storage.S3.S3Utils; |
| import org.apache.cloudstack.managed.context.ManagedContextRunnable; |
| import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; |
| import org.apache.commons.httpclient.Header; |
| import org.apache.commons.httpclient.HttpClient; |
| import org.apache.commons.httpclient.HttpStatus; |
| import org.apache.commons.httpclient.URIException; |
| import org.apache.commons.httpclient.methods.GetMethod; |
| import org.apache.commons.httpclient.params.HttpMethodParams; |
| import org.apache.commons.lang3.StringUtils; |
| |
| import java.io.BufferedInputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.util.Date; |
| import java.util.List; |
| |
| import static com.cloud.utils.NumbersUtil.toHumanReadableSize; |
| import static java.util.Arrays.asList; |
| |
| /** |
| * Download a template file using HTTP(S) |
| * |
| * This class, once instantiated, has the purpose to download a single Template to an S3 Image Store. |
| * |
| * Execution of the instance is started when runInContext() is called. |
| */ |
| public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader { |
| |
| private final String downloadUrl; |
| private final String s3Key; |
| private final String fileExtension; |
| private final HttpClient httpClient; |
| private final GetMethod getMethod; |
| private final DownloadCompleteCallback downloadCompleteCallback; |
| private final S3TO s3TO; |
| private String errorString = ""; |
| private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED; |
| private ResourceType resourceType = ResourceType.TEMPLATE; |
| private long remoteSize; |
| private long downloadTime; |
| private long totalBytes; |
| private long maxTemplateSizeInByte; |
| private boolean resume = false; |
| private boolean followRedirects = false; |
| |
| public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback, |
| long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) { |
| this.downloadUrl = downloadUrl; |
| this.s3TO = s3TO; |
| this.resourceType = resourceType; |
| this.maxTemplateSizeInByte = maxTemplateSizeInBytes; |
| this.httpClient = HTTPUtils.getHTTPClient(); |
| this.downloadCompleteCallback = downloadCompleteCallback; |
| |
| // Create a GET method for the download url. |
| this.getMethod = new GetMethod(downloadUrl); |
| |
| // Set the retry handler, default to retry 5 times. |
| this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5)); |
| |
| // Follow redirects |
| this.getMethod.setFollowRedirects(followRedirects); |
| |
| // Set file extension. |
| this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), "."); |
| |
| // Calculate and set S3 Key. |
| this.s3Key = StringUtils.join(asList(installPath, StringUtils.substringAfterLast(downloadUrl, "/")), S3Utils.SEPARATOR); |
| |
| // Set proxy if available. |
| HTTPUtils.setProxy(proxy, this.httpClient); |
| |
| // Set credentials if available. |
| HTTPUtils.setCredentials(username, password, this.httpClient); |
| } |
| |
| @Override |
| public long download(boolean resume, DownloadCompleteCallback callback) { |
| if (!status.equals(Status.NOT_STARTED)) { |
| // Only start downloading if we haven't started yet. |
| logger.debug("Template download is already started, not starting again. Template: " + downloadUrl); |
| |
| return 0; |
| } |
| |
| int responseCode; |
| if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) { |
| errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl; |
| logger.warn(errorString); |
| |
| status = Status.UNRECOVERABLE_ERROR; |
| return 0; |
| } |
| |
| boolean failedDueToRedirection = List.of(HttpStatus.SC_MOVED_PERMANENTLY, |
| HttpStatus.SC_MOVED_TEMPORARILY).contains(responseCode) && !followRedirects; |
| if (!HTTPUtils.verifyResponseCode(responseCode) || failedDueToRedirection) { |
| errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode; |
| logger.warn(errorString); |
| status = Status.UNRECOVERABLE_ERROR; |
| return 0; |
| } |
| |
| // Headers |
| Header contentLengthHeader = getMethod.getResponseHeader("content-length"); |
| Header contentTypeHeader = getMethod.getResponseHeader("content-type"); |
| |
| // Check the contentLengthHeader and transferEncodingHeader. |
| if (contentLengthHeader == null) { |
| errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied"; |
| logger.warn(errorString); |
| |
| status = Status.UNRECOVERABLE_ERROR; |
| return 0; |
| } else { |
| // The ContentLengthHeader is supplied, parse it's value. |
| remoteSize = Long.parseLong(contentLengthHeader.getValue()); |
| } |
| |
| if (remoteSize > maxTemplateSizeInByte) { |
| errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte; |
| logger.warn(errorString); |
| |
| status = Status.UNRECOVERABLE_ERROR; |
| return 0; |
| } |
| |
| InputStream inputStream; |
| |
| try { |
| inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream()); |
| } catch (IOException e) { |
| errorString = "Exception occurred while opening InputStream for template " + downloadUrl; |
| logger.warn(errorString); |
| |
| status = Status.UNRECOVERABLE_ERROR; |
| return 0; |
| } |
| |
| logger.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + toHumanReadableSize(remoteSize) + " bytes"); |
| |
| // Time the upload starts. |
| final Date start = new Date(); |
| |
| ObjectMetadata objectMetadata = new ObjectMetadata(); |
| objectMetadata.setContentLength(remoteSize); |
| |
| if (contentTypeHeader.getValue() != null) { |
| objectMetadata.setContentType(contentTypeHeader.getValue()); |
| } |
| |
| // Create the PutObjectRequest. |
| PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata); |
| |
| // If reduced redundancy is enabled, set it. |
| if (s3TO.isEnableRRS()) { |
| putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy); |
| } |
| |
| Upload upload = S3Utils.putObject(s3TO, putObjectRequest); |
| |
| upload.addProgressListener(new ProgressListener() { |
| @Override |
| public void progressChanged(ProgressEvent progressEvent) { |
| |
| // Record the amount of bytes transferred. |
| totalBytes += progressEvent.getBytesTransferred(); |
| |
| logger.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds"); |
| |
| if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) { |
| status = Status.IN_PROGRESS; |
| } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { |
| status = Status.DOWNLOAD_FINISHED; |
| } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) { |
| status = Status.ABORTED; |
| } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) { |
| status = Status.UNRECOVERABLE_ERROR; |
| } |
| } |
| }); |
| |
| try { |
| // Wait for the upload to complete. |
| upload.waitForCompletion(); |
| } catch (InterruptedException e) { |
| // Interruption while waiting for the upload to complete. |
| logger.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete"); |
| } |
| |
| downloadTime = new Date().getTime() - start.getTime(); |
| |
| if (status == Status.DOWNLOAD_FINISHED) { |
| logger.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); |
| } else { |
| logger.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString()); |
| } |
| |
| // Close input stream |
| getMethod.releaseConnection(); |
| |
| // Call the callback! |
| if (callback != null) { |
| callback.downloadComplete(status); |
| } |
| |
| return totalBytes; |
| } |
| |
| public String getDownloadUrl() { |
| try { |
| return getMethod.getURI().toString(); |
| } catch (URIException e) { |
| return null; |
| } |
| } |
| |
| @Override |
| public Status getStatus() { |
| return status; |
| } |
| |
| @Override |
| public long getDownloadTime() { |
| return downloadTime; |
| } |
| |
| @Override |
| public long getDownloadedBytes() { |
| return totalBytes; |
| } |
| |
| /** |
| * Returns an InputStream only when the status is DOWNLOAD_FINISHED. |
| * |
| * The caller of this method must close the InputStream to prevent resource leaks! |
| * |
| * @return S3ObjectInputStream of the object. |
| */ |
| public InputStream getS3ObjectInputStream() { |
| // Check if the download is finished |
| if (status != Status.DOWNLOAD_FINISHED) { |
| return null; |
| } |
| |
| return S3Utils.getObjectStream(s3TO, s3TO.getBucketName(), s3Key); |
| } |
| |
| public void cleanupAfterError() { |
| logger.warn("Cleanup after error, trying to remove object: " + s3Key); |
| |
| S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key); |
| } |
| |
| @Override |
| public boolean stopDownload() { |
| switch (status) { |
| case IN_PROGRESS: |
| if (getMethod != null) { |
| getMethod.abort(); |
| } |
| break; |
| case UNKNOWN: |
| case NOT_STARTED: |
| case RECOVERABLE_ERROR: |
| case UNRECOVERABLE_ERROR: |
| case ABORTED: |
| case DOWNLOAD_FINISHED: |
| // Remove the object if it already has been uploaded. |
| S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key); |
| break; |
| default: |
| break; |
| } |
| |
| status = TemplateDownloader.Status.ABORTED; |
| return true; |
| } |
| |
| @Override |
| public int getDownloadPercent() { |
| if (remoteSize == 0) { |
| return 0; |
| } |
| |
| return (int) (100.0 * totalBytes / remoteSize); |
| } |
| |
| @Override |
| protected void runInContext() { |
| // Start the download! |
| download(resume, downloadCompleteCallback); |
| } |
| |
| @Override |
| public void setStatus(Status status) { |
| this.status = status; |
| } |
| |
| public boolean isResume() { |
| return resume; |
| } |
| |
| @Override |
| public String getDownloadError() { |
| return errorString; |
| } |
| |
| @Override |
| public String getDownloadLocalPath() { |
| return s3Key; |
| } |
| |
| @Override |
| public void setResume(boolean resume) { |
| this.resume = resume; |
| } |
| |
| @Override |
| public long getMaxTemplateSizeInBytes() { |
| return maxTemplateSizeInByte; |
| } |
| |
| @Override |
| public void setDownloadError(String error) { |
| errorString = error; |
| } |
| |
| @Override |
| public boolean isInited() { |
| return true; |
| } |
| |
| public ResourceType getResourceType() { |
| return resourceType; |
| } |
| |
| public long getTotalBytes() { |
| return totalBytes; |
| } |
| |
| public String getFileExtension() { |
| return fileExtension; |
| } |
| |
| @Override |
| public void setFollowRedirects(boolean followRedirects) { |
| this.followRedirects = followRedirects; |
| if (this.getMethod != null) { |
| this.getMethod.setFollowRedirects(followRedirects); |
| } |
| } |
| } |