Merge pull request #1256 from apache/DATALAB-408
[DATALAB-408]: [SSN] Added docker images to Nexus
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index e3bdd4c..f1442de 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -206,11 +206,21 @@
def build_docker_images(args):
- # Building base and ssn docker images
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
+ if args.conf_repository_user and args.conf_repository_pass and args.conf_repository_address:
+ subprocess.run( 'sudo docker login -u {0} -p {1} {2}:8083'
+ .format(args.conf_repository_user, args.conf_repository_pass, args.conf_repository_address), shell=True, check=True)
+ subprocess.run('sudo docker pull {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image tag {}:8083/docker.datalab-base-{} docker.datalab-base'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image rm {}:8083/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker pull {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image tag {}:8083/docker.datalab-ssn-{} docker.datalab-ssn'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image rm {}:8083/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_cloud_provider), shell=True, check=True)
+ else:
+ # Building base and ssn docker images
+ subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
'infrastructure-provisioning/src/general/files/{1}/'
'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
+ subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 7218cb9..ec351cf 100644
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@ -206,7 +206,7 @@
# INSTALL LIVY
if not exists(conn, '/home/{0}/.ensure_dir/livy_ensured'.format(args.os_user)):
conn.sudo('wget -P /tmp/ --user={} --password={} '
- '{}/repository/packages/livy.tar.gz --no-check-certificate'
+ 'https://{}/repository/packages/livy.tar.gz --no-check-certificate'
.format(os.environ['conf_repository_user'],
os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
conn.sudo('tar -xzvf /tmp/livy.tar.gz -C /tmp/')
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 844d0a7..e005f35 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -868,7 +868,7 @@
if exists(conn,'/usr/local/lib/livy'):
conn.sudo('rm -r /usr/local/lib/livy')
conn.sudo('wget -P /tmp/ --user={} --password={} '
- '{}/repository/packages/livy.tar.gz --no-check-certificate'
+ 'https://{}/repository/packages/livy.tar.gz --no-check-certificate'
.format(os.environ['conf_repository_user'],
os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
conn.sudo('tar -xzvf /tmp/livy.tar.gz -C /usr/local/lib/')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index aa20a68..849bce0 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -113,28 +113,42 @@
def build_docker_images(image_list, region, datalab_path):
try:
- host_string = '{}@{}'.format(args.os_user, args.hostname)
- if os.environ['conf_cloud_provider'] == 'azure':
- conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
- 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
- conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
- '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
- if region == 'cn-north-1':
- add_china_repository(datalab_path)
- for image in image_list:
- name = image['name']
- tag = image['tag']
- conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
- '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
- if name == 'base':
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
- ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- else:
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
- return True
+ if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ:
+ conn.sudo('sudo docker login -u {0} -p {1} {2}:8083'
+ .format(os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('docker pull {0}:8083/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
+ conn.sudo('docker image tag {0}:8083/docker.datalab-{2}-{1}:{3} docker.datalab-{2}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
+ conn.sudo('docker image rm {0}:8083/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag))
+ return True
+ else:
+ host_string = '{}@{}'.format(args.os_user, args.hostname)
+ if os.environ['conf_cloud_provider'] == 'azure':
+ conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
+ 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
+ conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
+ '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
+ if region == 'cn-north-1':
+ add_china_repository(datalab_path)
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
+ '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
+ if name == 'base':
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
+ ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ else:
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
+ return True
except:
return False
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index 7a35f90..78e0eb9 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -33,6 +33,7 @@
from fabric import *
from patchwork.files import exists
from patchwork import files
+import time
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
@@ -153,7 +154,8 @@
if args.cloud_provider == 'azure' and os.environ['azure_datalake_enable'] == 'true':
conn.sudo('sed -i "s|\'use_ldap\': true|{}|g" ' + args.datalab_path + 'sources/services/self-service/src/main/resources/webapp/src/dictionary/azure.dictionary.ts'.format(
'\'use_ldap\': false'))
-
+ conn.sudo('rm -rf {}sources/services/self-service/src/main/resources/webapp/node_modules'.format(
+ args.datalab_path))
conn.sudo('bash -c "cd {}sources/services/self-service/src/main/resources/webapp/ && echo "N" | npm install"'.format(args.datalab_path))
manage_npm_pkg('bash -c "cd {}sources/services/self-service/src/main/resources/webapp/ && npm run build.prod"'.format(args.datalab_path))
conn.sudo('sudo chown -R {} {}/*'.format(args.os_user, args.datalab_path))
@@ -161,14 +163,14 @@
# Building Back-end
if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ:
conn.sudo(
- 'wget -P {0}sources/services/provisioning-service/target/ --user={1} --password={2} {3}/repository/packages/{4}/provisioning-service-{4}.jar --no-check-certificate'
+ 'wget -P {0}sources/services/provisioning-service/target/ --user={1} --password={2} https://{3}/repository/packages/{4}/provisioning-service-{4}.jar --no-check-certificate'
.format(args.datalab_path, os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address'], os.environ['conf_release_tag']))
conn.sudo(
- 'wget -P {0}sources/services/self-service/target/ --user={1} --password={2} {3}/repository/packages/{4}/self-service-{4}.jar --no-check-certificate'
+ 'wget -P {0}sources/services/self-service/target/ --user={1} --password={2} https://{3}/repository/packages/{4}/self-service-{4}.jar --no-check-certificate'
.format(args.datalab_path, os.environ['conf_repository_user'], os.environ['conf_repository_pass'],
os.environ['conf_repository_address'], os.environ['conf_release_tag']))
conn.sudo(
- 'wget -P {0}sources/services/billing-{4}/target/ --user={1} --password={2} {3}/repository/packages/{5}/billing-{4}-{5}.jar --no-check-certificate'
+ 'wget -P {0}sources/services/billing-{4}/target/ --user={1} --password={2} https://{3}/repository/packages/{5}/billing-{4}-{5}.jar --no-check-certificate'
.format(args.datalab_path, os.environ['conf_repository_user'], os.environ['conf_repository_pass'],
os.environ['conf_repository_address'], args.cloud_provider, os.environ['conf_release_tag']))
else:
