Merge pull request #1257 from apache/DATALAB-408
[DATALAB-408]: added repository port as a separate parameter, added possibility to enable or disable jars/docker images download from repository
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index f1442de..9eba556 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -167,6 +167,12 @@
help='password to access repository (used for jars download)')
parser.add_argument('--conf_repository_address', type=str, default='',
help='address to access repository (used for jars download)')
+parser.add_argument('--conf_repository_port', type=str, default='',
+ help='port to access repository (used for jars download)')
+parser.add_argument('--conf_download_jars', type=str, default='false',
+ help='whether to download jars from repository (True) or to build from sources (False)')
+parser.add_argument('--conf_download_docker_images', type=str, default='false',
+ help='whether to download docker images from repository (True) or to build from sources (False)')
parser.add_argument('--conf_letsencrypt_domain_name', type=str, default='', help='Domain names to apply. '
'For multiple domains enter a comma separated list of domains as a parameter'
'ssn.domain_name will be used for ssn_node, DNS A record have to exist during deployment')
@@ -206,15 +212,15 @@
def build_docker_images(args):
- if args.conf_repository_user and args.conf_repository_pass and args.conf_repository_address:
- subprocess.run( 'sudo docker login -u {0} -p {1} {2}:8083'
- .format(args.conf_repository_user, args.conf_repository_pass, args.conf_repository_address), shell=True, check=True)
- subprocess.run('sudo docker pull {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
- subprocess.run('sudo docker image tag {}:8083/docker.datalab-base-{} docker.datalab-base'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
- subprocess.run('sudo docker image rm {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
- subprocess.run('sudo docker pull {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
- subprocess.run('sudo docker image tag {}:8083/docker.datalab-ssn-{} docker.datalab-ssn'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
- subprocess.run('sudo docker image rm {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
+ if args.conf_repository_user and args.conf_repository_pass and args.conf_repository_port and args.conf_repository_address and args.conf_download_docker_images == 'true':
+ subprocess.run( 'sudo docker login -u {0} -p {1} {2}:{3}'
+ .format(args.conf_repository_user, args.conf_repository_pass, args.conf_repository_address, args.conf_repository_port), shell=True, check=True)
+ subprocess.run('sudo docker pull {}:{}/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image tag {}:{}/docker.datalab-base-{} docker.datalab-base'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image rm {}:{}/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker pull {}:{}/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image tag {}:{}/docker.datalab-ssn-{} docker.datalab-ssn'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image rm {}:{}/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
else:
# Building base and ssn docker images
subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index f9d2d1b..9b049e6 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -91,6 +91,12 @@
# repository_pass =
### Repository url
# repository_address =
+### Repository port
+# repository_port =
+### download_jars enabled
+# download_jars =
+### download_docker_images enabled
+# download_docker_images =
### release tag
# release_tag =
### Deeplearning native cloud AMI enabled
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 849bce0..3a1f062 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -113,28 +113,28 @@
def build_docker_images(image_list, region, datalab_path):
try:
- if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ:
- conn.sudo('sudo docker login -u {0} -p {1} {2}:8083'
- .format(os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
+ host_string = '{}@{}'.format(args.os_user, args.hostname)
+ if os.environ['conf_cloud_provider'] == 'azure':
+ conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
+ 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
+ conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
+ '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
+ if region == 'cn-north-1':
+ add_china_repository(datalab_path)
+ if 'conf_repository_user' in os.environ and 'conf_repository_port' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ and os.environ['conf_download_docker_images'] == 'true':
+ conn.sudo('sudo docker login -u {0} -p {1} {2}:{3}'
+ .format(os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address'], os.environ['conf_repository_port']))
for image in image_list:
name = image['name']
tag = image['tag']
- conn.sudo('docker pull {0}:8083/docker.datalab-{2}-{1}:{3}'
- .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
- conn.sudo('docker image tag {0}:8083/docker.datalab-{2}-{1}:{3} docker.datalab-{2}:{3}'
- .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
- conn.sudo('docker image rm {0}:8083/docker.datalab-{2}-{1}:{3}'
- .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
+ conn.sudo('docker pull {0}:{4}/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ conn.sudo('docker image tag {0}:{4}/docker.datalab-{2}-{1}:{3} docker.datalab-{2}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ conn.sudo('docker image rm {0}:{4}/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
return True
else:
- host_string = '{}@{}'.format(args.os_user, args.hostname)
- if os.environ['conf_cloud_provider'] == 'azure':
- conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
- 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
- conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
- '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
- if region == 'cn-north-1':
- add_china_repository(datalab_path)
for image in image_list:
name = image['name']
tag = image['tag']
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index 78e0eb9..d9327aa 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -161,7 +161,7 @@
conn.sudo('sudo chown -R {} {}/*'.format(args.os_user, args.datalab_path))
# Building Back-end
- if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ:
+ if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ and os.environ['conf_download_jars'] == 'true':
conn.sudo(
'wget -P {0}sources/services/provisioning-service/target/ --user={1} --password={2} https://{3}/repository/packages/{4}/provisioning-service-{4}.jar --no-check-certificate'
.format(args.datalab_path, os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address'], os.environ['conf_release_tag']))
