| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """ |
| HTTP utilities. |
| """ |
| |
| import os |
| import tempfile |
| |
| import requests |
| |
| |
| def download_file(url, destination=None, logger=None, progress_handler=None): |
| """ |
| Download file. |
| |
| :param url: URL from which to download |
| :type url: basestring |
| :param destination: path where the file should be saved or ``None`` to auto-generate |
| :type destination: basestring |
| :returns: path where the file was saved |
| :rtype: basestring |
| :raises exceptions.IOError: |
| :raises requests.exceptions.RequestException: |
| """ |
| chunk_size = 1024 |
| |
| if not destination: |
| file_descriptor, destination = tempfile.mkstemp() |
| os.close(file_descriptor) |
| if logger: |
| logger.info('Downloading {0} to {1}...'.format(url, destination)) |
| |
| response = requests.get(url, stream=True) |
| final_url = response.url |
| if final_url != url and logger: |
| logger.debug('Redirected to {0}'.format(final_url)) |
| |
| read_bytes = 0 |
| total_size = int(response.headers['Content-Length']) \ |
| if 'Content-Length' in response.headers else None |
| try: |
| with open(destination, 'wb') as destination_file: |
| for chunk in response.iter_content(chunk_size): |
| destination_file.write(chunk) |
| if total_size and progress_handler: |
| # Only showing progress bar if we have the total content length |
| read_bytes += chunk_size |
| progress_handler(read_bytes, total_size) |
| finally: |
| response.close() |
| |
| return destination |