| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| import contextlib |
| import logging |
| import logging.config |
| import os |
| import subprocess |
| import sys |
| |
| import requests |
| |
| |
| def get_mxnet_root() -> str: |
| curpath = os.path.abspath(os.path.dirname(__file__)) |
| |
| def is_mxnet_root(path: str) -> bool: |
| return os.path.exists(os.path.join(path, ".mxnet_root")) |
| |
| while not is_mxnet_root(curpath): |
| parent = os.path.abspath(os.path.join(curpath, os.pardir)) |
| if parent == curpath: |
| raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root") |
| curpath = parent |
| return curpath |
| |
| |
| @contextlib.contextmanager |
| def remember_cwd(): |
| ''' |
| Restore current directory when exiting context |
| ''' |
| curdir = os.getcwd() |
| try: yield |
| finally: os.chdir(curdir) |
| |
| |
| def retry(target_exception, tries=4, delay_s=1, backoff=2): |
| """Retry calling the decorated function using an exponential backoff. |
| |
| http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ |
| original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry |
| |
| :param target_exception: the exception to check. may be a tuple of |
| exceptions to check |
| :type target_exception: Exception or tuple |
| :param tries: number of times to try (not retry) before giving up |
| :type tries: int |
| :param delay_s: initial delay between retries in seconds |
| :type delay_s: int |
| :param backoff: backoff multiplier e.g. value of 2 will double the delay |
| each retry |
| :type backoff: int |
| """ |
| import time |
| from functools import wraps |
| |
| def decorated_retry(f): |
| @wraps(f) |
| def f_retry(*args, **kwargs): |
| mtries, mdelay = tries, delay_s |
| while mtries > 1: |
| try: |
| return f(*args, **kwargs) |
| except target_exception as e: |
| logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay) |
| time.sleep(mdelay) |
| mtries -= 1 |
| mdelay *= backoff |
| return f(*args, **kwargs) |
| |
| return f_retry # true decorator |
| |
| return decorated_retry |
| |
| |
| # noinspection SyntaxError |
| def under_ci() -> bool: |
| """:return: True if we run in Jenkins.""" |
| return 'JOB_NAME' in os.environ |
| |
| |
| def ec2_instance_info() -> str: |
| import requests |
| urls = [ |
| "http://instance-data/latest/meta-data/instance-type", |
| "http://instance-data/latest/meta-data/instance-id", |
| "http://instance-data/latest/meta-data/public-hostname", |
| "http://instance-data/latest/meta-data/ami-id" |
| ] |
| if under_ci(): |
| result = [] |
| try: |
| for url in urls: |
| r = requests.get(url) |
| if r.status_code == 200: |
| result.append(r.content.decode()) |
| return ' '.join(result) |
| except ConnectionError: |
| pass |
| return '?' |
| else: |
| return '' |
| |
| |
| def chdir_to_script_directory(): |
| # We need to be in the same directory than the script so the commands in the dockerfiles work as |
| # expected. But the script can be invoked from a different path |
| base = os.path.split(os.path.realpath(__file__))[0] |
| os.chdir(base) |
| |
| |
| def script_name() -> str: |
| """:returns: script name with leading paths removed""" |
| return os.path.split(sys.argv[0])[1] |
| |
| |
| def config_logging(): |
| conf_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logging.conf") |
| logging.config.fileConfig(os.getenv('LOGGING_CONF', conf_path)) |
| |
| # Force botocore and requests are set to WARNING to avoid leaking any credentials |
| # or sensitive information |
| logging.getLogger("botocore").setLevel(logging.WARNING) |
| logging.getLogger("requests").setLevel(logging.WARNING) |
| |
| |
| # Takes url and downloads it to the dest_path directory on Windows. |
| def download_file(url, dest_path): |
| file_name = url.split('/')[-1] |
| full_path = "{}\\{}".format(dest_path, file_name) |
| logging.info("Downloading: {}".format(full_path)) |
| r = requests.get(url, stream=True) |
| if r.status_code == 404: |
| return r.status_code |
| elif r.status_code != 200: |
| logging.error("{} returned status code {}".format(url, r.status_code)) |
| with open(full_path, 'wb') as f: |
| for chunk in r.iter_content(chunk_size=1024): |
| if chunk: # filter out keep-alive new chunks |
| f.write(chunk) |
| return full_path |
| |
| |
| # Takes arguments and runs command on host. Shell is disabled by default. |
| def run_command(args, shell=False): |
| try: |
| logging.info("Issuing command: {}".format(args)) |
| res = subprocess.check_output(args, shell=shell, timeout=1800).decode("utf-8").replace("\r\n", "") |
| logging.info("Output: {}".format(res)) |
| except subprocess.CalledProcessError as e: |
| raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) |
| return res |