Merge branch 'DATALAB-2091' into DATALAB-2258
diff --git a/README.md b/README.md
index 33721fb..24d52b5 100644
--- a/README.md
+++ b/README.md
@@ -2432,7 +2432,7 @@
COPY ssn/ /root/
COPY general/scripts/aws/ssn_* /root/scripts/
-COPY general/lib/os/${OS}/ssn_lib.py /usr/lib/python2.7/datalab/ssn_lib.py
+COPY general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY general/files/aws/ssn_policy.json /root/files/
COPY general/templates/aws/jenkins_jobs /root/templates/jenkins_jobs
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 530f19b..2e8b7e0 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
diff --git a/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py b/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py
index dce8a86..1f1922d 100644
--- a/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py
+++ b/infrastructure-provisioning/scripts/deploy_keycloak/deploy_keycloak.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index 0d05532..c6f8d5e 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
@@ -1257,7 +1257,7 @@
'stable"')
sudo('apt-get update')
sudo('apt-cache policy docker-ce')
- sudo('apt-get install -y docker-ce={}~ce-0~ubuntu'.format(configuration['ssn_docker_version']))
+ sudo('apt-get install -y docker-ce=5:{}~3-0~ubuntu-focal'.format(configuration['ssn_docker_version']))
sudo('usermod -a -G docker ' + configuration['conf_os_user'])
sudo('update-rc.d docker defaults')
sudo('update-rc.d docker enable')
diff --git a/infrastructure-provisioning/scripts/deploy_repository/files/Dockerfile b/infrastructure-provisioning/scripts/deploy_repository/files/Dockerfile
index af2c039..8b3d049 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/files/Dockerfile
+++ b/infrastructure-provisioning/scripts/deploy_repository/files/Dockerfile
@@ -24,9 +24,9 @@
# Install any .deb dependecies
RUN apt-get update && \
apt-get -y upgrade && \
- apt-get -y install python-pip python-dev groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
+ apt-get -y install python3-pip python3-dev groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install any python dependencies
RUN pip install -UI pip==9.0.3
-RUN pip install boto3 backoff fabric==1.14.0 fabvenv awscli argparse ujson jupyter pycrypto
\ No newline at end of file
+RUN pip install boto3 backoff fabric==1.14.0 fabvenv awscli argparse ujson jupyter pycryptodome
\ No newline at end of file
diff --git a/infrastructure-provisioning/scripts/deploy_repository/scripts/update_amazon_repositories.py b/infrastructure-provisioning/scripts/deploy_repository/scripts/update_amazon_repositories.py
index df8e20f..609cc2c 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/scripts/update_amazon_repositories.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/scripts/update_amazon_repositories.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
diff --git a/infrastructure-provisioning/scripts/jenkins/sonar.py b/infrastructure-provisioning/scripts/jenkins/sonar.py
index db4c5ea..f5846cd 100644
--- a/infrastructure-provisioning/scripts/jenkins/sonar.py
+++ b/infrastructure-provisioning/scripts/jenkins/sonar.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
diff --git a/infrastructure-provisioning/scripts/post-deployment_configuration.py b/infrastructure-provisioning/scripts/post-deployment_configuration.py
index 7a39190..4df4e15 100644
--- a/infrastructure-provisioning/scripts/post-deployment_configuration.py
+++ b/infrastructure-provisioning/scripts/post-deployment_configuration.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/base/entrypoint.py b/infrastructure-provisioning/src/base/entrypoint.py
index 4780acf..d00742d 100644
--- a/infrastructure-provisioning/src/base/entrypoint.py
+++ b/infrastructure-provisioning/src/base/entrypoint.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -22,7 +22,7 @@
# ******************************************************************************
import os
-from ConfigParser import SafeConfigParser
+from configparser import SafeConfigParser
import argparse
from fabric.api import *
import json
diff --git a/infrastructure-provisioning/src/base/scripts/create_ssh_user.py b/infrastructure-provisioning/src/base/scripts/create_ssh_user.py
index 482eb2c..fb9dfbb 100644
--- a/infrastructure-provisioning/src/base/scripts/create_ssh_user.py
+++ b/infrastructure-provisioning/src/base/scripts/create_ssh_user.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
index 5c95894..88611ee 100644
--- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
+++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -33,7 +33,7 @@
parser.add_argument('--hostname', type=str, default='')
parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--pip_packages', type=str,
- default='boto3 argparse fabric==1.14.0 awscli google-api-python-client google-auth-httplib2 google-cloud-storage pycrypto azure==2.0.0')
+ default='boto3 argparse fab-classic awscli google-api-python-client google-auth-httplib2 google-cloud-storage pycryptodome azure==2.0.0')
parser.add_argument('--additional_config', type=str, default='{"empty":"string"}')
parser.add_argument('--user', type=str, default='')
parser.add_argument('--edge_private_ip', type=str, default='')
diff --git a/infrastructure-provisioning/src/base/scripts/install_user_key.py b/infrastructure-provisioning/src/base/scripts/install_user_key.py
index 74e22de..baace81 100644
--- a/infrastructure-provisioning/src/base/scripts/install_user_key.py
+++ b/infrastructure-provisioning/src/base/scripts/install_user_key.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/dataengine-service/fabfile.py b/infrastructure-provisioning/src/dataengine-service/fabfile.py
index 8f475b4..a508540 100644
--- a/infrastructure-provisioning/src/dataengine-service/fabfile.py
+++ b/infrastructure-provisioning/src/dataengine-service/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/dataengine/fabfile.py b/infrastructure-provisioning/src/dataengine/fabfile.py
index f5775a0..752f8e4 100644
--- a/infrastructure-provisioning/src/dataengine/fabfile.py
+++ b/infrastructure-provisioning/src/dataengine/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 1d299a1..8b576a7 100644
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -51,7 +51,6 @@
caffe2_version = os.environ['notebook_caffe2_version']
cmake_version = os.environ['notebook_cmake_version']
cntk_version = os.environ['notebook_cntk_version']
-cntk2_version = os.environ['notebook_cntk2_version']
mxnet_version = os.environ['notebook_mxnet_version']
python3_version = "3.4"
scala_link = "http://www.scala-lang.org/files/archive/"
@@ -141,8 +140,6 @@
or os.environ['application'] in ('rstudio', 'tensor-rstudio'):
print("Installing R")
ensure_r(args.os_user, r_libs, args.region, args.r_mirror)
- print("Install Python 2 modules")
- ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
if os.environ['application'] == 'zeppelin':
@@ -175,7 +172,7 @@
#print("Installing Torch")
#install_torch(args.os_user)
print("Install CNTK Python library")
- install_cntk(args.os_user, cntk2_version, cntk_version)
+ install_cntk(args.os_user, cntk_version)
print("Installing MXNET")
install_mxnet(args.os_user, mxnet_version)
diff --git a/infrastructure-provisioning/src/deeplearning/fabfile.py b/infrastructure-provisioning/src/deeplearning/fabfile.py
index ad090d2..deaa516 100644
--- a/infrastructure-provisioning/src/deeplearning/fabfile.py
+++ b/infrastructure-provisioning/src/deeplearning/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index cfbfb6c..142ecc6 100644
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -55,7 +55,6 @@
caffe2_version = os.environ['notebook_caffe2_version']
cmake_version = os.environ['notebook_cmake_version']
cntk_version = os.environ['notebook_cntk_version']
-cntk2_version = os.environ['notebook_cntk2_version']
mxnet_version = os.environ['notebook_mxnet_version']
keras_version = os.environ['notebook_keras_version']
theano_version = os.environ['notebook_theano_version']
@@ -112,8 +111,6 @@
# INSTALL LANGUAGES
print("Install Java")
ensure_jre_jdk(args.os_user)
- print("Install Python 2 modules")
- ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
@@ -131,7 +128,7 @@
#print("Installing Torch")
#install_torch(args.os_user)
print("Install CNTK Python library")
- install_cntk(args.os_user,cntk2_version, cntk_version)
+ install_cntk(args.os_user, cntk_version)
print("Installing MXNET")
install_mxnet(args.os_user, mxnet_version)
diff --git a/infrastructure-provisioning/src/edge/fabfile.py b/infrastructure-provisioning/src/edge/fabfile.py
index 01ff3b0..7f3cedb 100644
--- a/infrastructure-provisioning/src/edge/fabfile.py
+++ b/infrastructure-provisioning/src/edge/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
index 0adb505..3d49a2d 100644
--- a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
+++ b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py b/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py
index 8b67a41..cefaffe 100644
--- a/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py
+++ b/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py b/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py
index bef9ec9..20dcd4c 100644
--- a/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py
+++ b/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/check_inactivity.py b/infrastructure-provisioning/src/general/api/check_inactivity.py
index e07609b..d18b07e 100644
--- a/infrastructure-provisioning/src/general/api/check_inactivity.py
+++ b/infrastructure-provisioning/src/general/api/check_inactivity.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/configure.py b/infrastructure-provisioning/src/general/api/configure.py
index 691f3db..0c0285a 100644
--- a/infrastructure-provisioning/src/general/api/configure.py
+++ b/infrastructure-provisioning/src/general/api/configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/create.py b/infrastructure-provisioning/src/general/api/create.py
index e0910dc..e92db83 100644
--- a/infrastructure-provisioning/src/general/api/create.py
+++ b/infrastructure-provisioning/src/general/api/create.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/create_image.py b/infrastructure-provisioning/src/general/api/create_image.py
index df85de9..eb4828a 100644
--- a/infrastructure-provisioning/src/general/api/create_image.py
+++ b/infrastructure-provisioning/src/general/api/create_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/git_creds.py b/infrastructure-provisioning/src/general/api/git_creds.py
index a5a7e1b..f9f2994 100644
--- a/infrastructure-provisioning/src/general/api/git_creds.py
+++ b/infrastructure-provisioning/src/general/api/git_creds.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/install_libs.py b/infrastructure-provisioning/src/general/api/install_libs.py
index 3f818a5..3abeeeb 100644
--- a/infrastructure-provisioning/src/general/api/install_libs.py
+++ b/infrastructure-provisioning/src/general/api/install_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/list_libs.py b/infrastructure-provisioning/src/general/api/list_libs.py
index 5346b27..45dd0eb 100644
--- a/infrastructure-provisioning/src/general/api/list_libs.py
+++ b/infrastructure-provisioning/src/general/api/list_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/reconfigure_spark.py b/infrastructure-provisioning/src/general/api/reconfigure_spark.py
index 3e53885..fac58d6 100644
--- a/infrastructure-provisioning/src/general/api/reconfigure_spark.py
+++ b/infrastructure-provisioning/src/general/api/reconfigure_spark.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/recreate.py b/infrastructure-provisioning/src/general/api/recreate.py
index 63742e1..b81ee4f 100644
--- a/infrastructure-provisioning/src/general/api/recreate.py
+++ b/infrastructure-provisioning/src/general/api/recreate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/reupload_key.py b/infrastructure-provisioning/src/general/api/reupload_key.py
index 7f9b29a..ac3eab5 100644
--- a/infrastructure-provisioning/src/general/api/reupload_key.py
+++ b/infrastructure-provisioning/src/general/api/reupload_key.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/start.py b/infrastructure-provisioning/src/general/api/start.py
index 94e92b1..d583d2d 100644
--- a/infrastructure-provisioning/src/general/api/start.py
+++ b/infrastructure-provisioning/src/general/api/start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/status.py b/infrastructure-provisioning/src/general/api/status.py
index 97a767d..e45957e 100644
--- a/infrastructure-provisioning/src/general/api/status.py
+++ b/infrastructure-provisioning/src/general/api/status.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/stop.py b/infrastructure-provisioning/src/general/api/stop.py
index 7cc0123..6a97339 100644
--- a/infrastructure-provisioning/src/general/api/stop.py
+++ b/infrastructure-provisioning/src/general/api/stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/terminate.py b/infrastructure-provisioning/src/general/api/terminate.py
index 2ba94bb..f10b30d 100644
--- a/infrastructure-provisioning/src/general/api/terminate.py
+++ b/infrastructure-provisioning/src/general/api/terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/api/terminate_image.py b/infrastructure-provisioning/src/general/api/terminate_image.py
index d88d3bc..91a47dd 100644
--- a/infrastructure-provisioning/src/general/api/terminate_image.py
+++ b/infrastructure-provisioning/src/general/api/terminate_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 91249b0..200be2c 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -54,8 +54,8 @@
vpc2_cidr = '172.32.0.0/16'
### Enable or disable duo VPC mode(true|false)
duo_vpc_enable = false
-### Range of subnets which will be using for user's environments
-# user_subnets_range =
+### Range of subnets which will be used for user's environments
+# user_subnets_range = 172.31.0.0/24 - 172.31.50.0/24
### Comma-separated CIDR of IPs which will have access to SSN and Edge nodes
allowed_ip_cidr = '0.0.0.0/0'
### Type of network. Define in which network DataLab will be deployed. Possible options: public|private
@@ -85,8 +85,6 @@
# letsencrypt_email =
### Prefix of the private subnet
private_subnet_prefix = 24
-### Range of subnets defined by user
-# user_subnets_range = 172.31.0.0/24 - 172.31.50.0/24
#--- [aws] section contains all common parameters related to Amazon ---#
[aws]
@@ -119,7 +117,7 @@
### Amazon zone letter for ssn, edge and notebook subnet provisioning
# zone =
### Amazon ami name based on debian conf_os_family for all DataLab instances
-debian_image_name = ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200112
+debian_image_name = ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20201026
### Amazon ami name based on RedHat conf_os_family for all DataLab instances
redhat_image_name = RHEL-7.4_HVM-20180103-x86_64-2-Hourly2-GP2
### Amazon account ID
@@ -153,7 +151,7 @@
### Slave node size for Data Engine
# dataengine_slave_size =
### Azure image name based on debian conf_os_family for all DataLab instances
-debian_image_name = Canonical_UbuntuServer_18.04-LTS
+debian_image_name = Canonical_UbuntuServer_20.04-LTS
### Azure image name based on RedHat conf_os_family for all DataLab instances
redhat_image_name = RedHat_RHEL_7.3
### Azure AD user name
@@ -191,7 +189,7 @@
### GCP zone name for whole DataLab provisioning
zone = us-west1-a
### GCP ami name based on debian conf_os_family for all DataLab instances
-debian_image_name = /projects/ubuntu-os-cloud/global/images/ubuntu-1804-bionic-v20200317
+debian_image_name = /projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20210119a
### GCP ami name based on RedHat conf_os_family for all DataLab instances
redhat_image_name =
### Prefix of the private subnet
@@ -209,7 +207,7 @@
### Elastic IP which will be associated with SSN node
# elastic_ip =
### Version of Docker to be installed on SSN
-docker_version = 18.06.3
+docker_version = 20.10.2
### Name of hosted zone for Route53
# hosted_zone_name =
### ID of hosted zone
@@ -239,13 +237,13 @@
### Version of Apache Hadoop to be installed on notebook
hadoop_version = 2.7
### Version of Jupyter to be installed on notebook
-jupyter_version = 6.0.2
+jupyter_version = 6.1.6
### Version of TensorFlow to be installed on notebook
-tensorflow_version = 2.1.0
+tensorflow_version = 2.3.2
### Version of Zeppelin to be installed on notebook
-zeppelin_version = 0.8.2
+zeppelin_version = 0.9.0
### Version of Rstudio to be installed on notebook
-rstudio_version = 1.2.5033
+rstudio_version = 1.4.1103-4
### Version of Scala to be installed on notebook
scala_version = 2.12.8
### Version of Livy top be installed on notebook
@@ -264,9 +262,6 @@
pytorch_branch = release/1.5
### Cmake version for DeepLearning notebook
cmake_version = 3.15.5
-### CNTK library version for DeepLearning notebook for python2
-### All releases 2.4+ officially only support Ubuntu 16.04.
-cntk2_version = 2.5
### CNTK library version for DeepLearning notebook for python3
cntk_version = 2.7
### MXNet library version for DeepLearning notebook for python
@@ -289,11 +284,9 @@
tornado_version = 5.1.1
ipykernel_version = 4.8.2
### Version of ungit if previous needed. Use latest as default.
-ungit_version = 1.4.36
+ungit_version = 1.5.15
### Numpy version
numpy_version = 1.18.3
-### Numpy version for python 2
-numpy2_version = 1.16.6
### caTools version for R 3.4.4
catools_version = 1.17.1.4
### Apache Ivy version
diff --git a/infrastructure-provisioning/src/general/files/aws/base_Dockerfile b/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
index fa83551..bcab7e0 100644
--- a/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
@@ -26,12 +26,12 @@
# Install any .deb dependecies
RUN apt-get update && \
apt-get -y upgrade && \
- apt-get -y install python-pip python-dev groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
+ apt-get -y install python3-pip python3-dev groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install any python dependencies
RUN pip install -UI qtconsole==4.7.7 pip==20.1 && \
- pip install boto3 backoff fabric==1.14.0 fabvenv awscli argparse requests ujson jupyter pycrypto
+ pip install boto3 backoff fabric==1.14.0 fabvenv awscli argparse requests ujson jupyter pycryptodome
# Configuring ssh for user
RUN mkdir -p /root/.ssh; echo "Host *" > /root/.ssh/config; \
@@ -58,16 +58,16 @@
mkdir -p /root/scripts && \
mkdir -p /root/templates && \
mkdir -p /root/files && \
- mkdir -p /usr/lib/python2.7/datalab && \
+ mkdir -p /usr/lib/python3.8/datalab && \
mkdir -p /root/keys/.ssh
COPY ${SRC_PATH}base/ /root
COPY ${SRC_PATH}general/conf/* /root/conf/
COPY ${SRC_PATH}general/api/*.py /bin/
COPY ${SRC_PATH}general/scripts/aws/common_* /root/scripts/
-COPY ${SRC_PATH}general/lib/aws/* /usr/lib/python2.7/datalab/
-COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python2.7/datalab/common_lib.py
-COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python2.7/datalab/fab.py
+COPY ${SRC_PATH}general/lib/aws/* /usr/lib/python3.8/datalab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
diff --git a/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile b/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
index 9a2296c..5cbefa5 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
@@ -30,8 +30,8 @@
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
COPY general/scripts/os/common_* /root/scripts/
-COPY general/lib/os/redhat/common_lib.py /usr/lib/python2.7/datalab/common_lib.py
-COPY general/lib/os/redhat/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/redhat/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY general/lib/os/redhat/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/inactive.sh /root/templates/
COPY general/templates/os/inactive.service /root/templates/
COPY general/templates/os/inactive.timer /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile b/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
index 0277816..5af2a5c 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
@@ -29,7 +29,7 @@
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/common_* /root/scripts/
COPY general/scripts/aws/dataengine_* /root/scripts/
COPY general/scripts/os/notebook_reconfigure_dataengine_spark.py /root/scripts/
diff --git a/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile b/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
index ce863e8..0df60c4 100644
--- a/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
@@ -27,7 +27,7 @@
COPY deeplearning/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/deeplearning_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile b/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile
index 7289921..c31e48e 100644
--- a/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile
@@ -26,7 +26,7 @@
COPY edge/ /root/
COPY general/scripts/aws/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python2.7/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/templates/aws/edge_s3_policy.json /root/templates/edge_s3_policy.json
RUN chmod a+x /root/fabfile.py; \
diff --git a/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile b/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
index 108e4cb..8312b72 100644
--- a/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
@@ -27,7 +27,7 @@
COPY jupyter/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/jupyter_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile b/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile
index 7daaa5c..6c4f0aa 100644
--- a/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile
@@ -27,7 +27,7 @@
COPY jupyterlab/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/jupyter* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/pyspark_local_template.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/project_Dockerfile b/infrastructure-provisioning/src/general/files/aws/project_Dockerfile
index b2baad7..5390023 100644
--- a/infrastructure-provisioning/src/general/files/aws/project_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/project_Dockerfile
@@ -27,7 +27,7 @@
COPY project/ /root/
COPY general/scripts/aws/project_* /root/scripts/
COPY general/scripts/aws/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python2.7/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/templates/aws/edge_s3_policy.json /root/templates/edge_s3_policy.json
COPY general/templates/os/manage_step_certs.sh /root/templates/
COPY general/templates/os/step-cert-manager.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
index abd2282..93041d3 100644
--- a/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
@@ -27,7 +27,7 @@
COPY rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/inactive_rs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile b/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
index ce3e3cf..b9c1548 100644
--- a/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
@@ -25,7 +25,7 @@
COPY infrastructure-provisioning/src/ssn/ /root/
COPY infrastructure-provisioning/src/general/scripts/aws/ssn_* /root/scripts/
-COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python2.7/datalab/ssn_lib.py
+COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY infrastructure-provisioning/src/general/files/aws/ssn_policy.json /root/files/
COPY infrastructure-provisioning/src/general/templates/aws/jenkins_jobs /root/templates/jenkins_jobs
COPY infrastructure-provisioning/src/general/templates/os/manage_step_certs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
index d24e1aa..5c7e95b 100644
--- a/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
@@ -27,7 +27,7 @@
COPY tensor-rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/tensor-rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
index 22eecb9..6bd820c 100644
--- a/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
@@ -27,7 +27,7 @@
COPY tensor/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/tensor_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile b/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
index c6fd9ef..abcb89e 100644
--- a/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
@@ -27,7 +27,7 @@
COPY zeppelin/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/zeppelin_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/aws/interpreter_livy.json /root/templates/
COPY general/templates/aws/interpreter_spark.json /root/templates/
COPY general/templates/aws/dataengine-service_interpreter_* /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
index b3a4d26..d7e5f21 100644
--- a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
@@ -26,12 +26,12 @@
# Install any .deb dependecies
RUN apt-get update && \
apt-get -y upgrade && \
- apt-get -y install python-pip python-dev groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
+ apt-get -y install python3-pip python3-dev groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install any python dependencies
RUN pip install -UI qtconsole==4.7.7 pip==20.1 && \
- pip install backoff fabric==1.14.0 fabvenv argparse requests ujson jupyter pycrypto azure==2.0.0 azure-mgmt-authorization pyyaml
+ pip install backoff fabric==1.14.0 fabvenv argparse requests ujson jupyter pycryptodome azure==2.0.0 azure-mgmt-authorization pyyaml
# Configuring ssh for user
RUN mkdir -p /root/.ssh; echo "Host *" > /root/.ssh/config; \
@@ -58,16 +58,16 @@
mkdir -p /root/scripts && \
mkdir -p /root/templates && \
mkdir -p /root/files && \
- mkdir -p /usr/lib/python2.7/datalab && \
+ mkdir -p /usr/lib/python3.8/datalab && \
mkdir -p /root/keys/.ssh
COPY ${SRC_PATH}base/ /root
COPY ${SRC_PATH}general/conf/* /root/conf/
COPY ${SRC_PATH}general/api/*.py /bin/
COPY ${SRC_PATH}general/scripts/azure/common_* /root/scripts/
-COPY ${SRC_PATH}general/lib/azure/* /usr/lib/python2.7/datalab/
-COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python2.7/datalab/common_lib.py
-COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python2.7/datalab/fab.py
+COPY ${SRC_PATH}general/lib/azure/* /usr/lib/python3.8/datalab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
diff --git a/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile b/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
index 4ed38a9..d1eec41 100644
--- a/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
@@ -24,7 +24,7 @@
ARG OS
COPY dataengine/ /root/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/dataengine_* /root/scripts/
COPY general/scripts/os/update_inactivity_on_start.py /root/scripts/
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
diff --git a/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile b/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile
index 6323abe..a9d548f 100644
--- a/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile
@@ -27,7 +27,7 @@
COPY deeplearning/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/deeplearning_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile b/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile
index 6ee207c..eeb5215 100644
--- a/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile
@@ -26,7 +26,7 @@
COPY edge/ /root/
COPY general/scripts/azure/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python2.7/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile b/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile
index 85c1c44..821ec24 100644
--- a/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile
@@ -27,7 +27,7 @@
COPY jupyter/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/jupyter_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile b/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile
index b2f392f..4fd212b 100644
--- a/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile
@@ -27,7 +27,7 @@
COPY jupyterlab/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/jupyter* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/project_Dockerfile b/infrastructure-provisioning/src/general/files/azure/project_Dockerfile
index a60747b..9f13f4b 100644
--- a/infrastructure-provisioning/src/general/files/azure/project_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/project_Dockerfile
@@ -27,7 +27,7 @@
COPY project/ /root/
COPY general/scripts/azure/project_* /root/scripts/
COPY general/scripts/azure/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python2.7/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/templates/os/manage_step_certs.sh /root/templates/
COPY general/templates/os/step-cert-manager.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile
index 8286ac1..0c39847 100644
--- a/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile
@@ -27,7 +27,7 @@
COPY rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/inactive_rs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile b/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile
index f257af0..8eb5885 100644
--- a/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile
@@ -25,7 +25,7 @@
COPY infrastructure-provisioning/src/ssn/ /root/
COPY infrastructure-provisioning/src/general/scripts/azure/ssn_* /root/scripts/
-COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python2.7/datalab/ssn_lib.py
+COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY infrastructure-provisioning/src/general/templates/azure/jenkins_jobs /root/templates/jenkins_jobs
COPY infrastructure-provisioning/src/general/templates/os/manage_step_certs.sh /root/templates/
COPY infrastructure-provisioning/src/general/templates/os/step-cert-manager.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
index 56aa7c5..4bb12ee 100644
--- a/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
@@ -27,7 +27,7 @@
COPY tensor/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/tensor_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile b/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile
index c9b0181..b2a0a6f 100644
--- a/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile
@@ -27,7 +27,7 @@
COPY zeppelin/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/zeppelin_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/azure/interpreter_livy.json /root/templates/
COPY general/templates/azure/interpreter_spark.json /root/templates/
COPY general/templates/os/dataengine_interpreter_livy.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
index 0590753..5203b6c 100644
--- a/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
@@ -19,19 +19,30 @@
#
# ******************************************************************************
-FROM ubuntu:18.04
+FROM ubuntu:20.04
ARG OS
ARG SRC_PATH
# Install any .deb dependecies
RUN apt-get update && \
apt-get -y upgrade && \
- apt-get -y install python-pip python-dev groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
+ DEBIAN_FRONTEND=noninteractive apt-get -y install python3-pip python3-dev groff vim less git wget nano libssl-dev libffi-dev libffi7 && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+# To cahnge POSIX locale to en_US.UTF-8
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y locales
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+ dpkg-reconfigure --frontend=noninteractive locales && \
+ update-locale LANG=en_US.UTF-8
+
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
# Install any python dependencies
-RUN pip install -UI qtconsole==4.7.7 pip==20.1 && \
- pip install boto3 backoff fabric==1.14.0 fabvenv argparse ujson jupyter pycrypto google-api-python-client google-cloud-storage \
+RUN python3 -m pip install -UI pip==20.1 && \
+ python3 -m pip install -U six configparser boto3 backoff fab-classic fabvenv argparse ujson jupyter pycryptodome google-api-python-client google-cloud-storage \
pyyaml google-auth-httplib2 oauth2client
# Configuring ssh for user
@@ -58,16 +69,16 @@
mkdir -p /root/scripts && \
mkdir -p /root/templates && \
mkdir -p /root/files && \
- mkdir -p /usr/lib/python2.7/datalab && \
+ mkdir -p /usr/lib/python3.8/datalab && \
mkdir -p /root/keys/.ssh
COPY ${SRC_PATH}base/ /root
COPY ${SRC_PATH}general/conf/* /root/conf/
COPY ${SRC_PATH}general/api/*.py /bin/
COPY ${SRC_PATH}general/scripts/gcp/common_* /root/scripts/
-COPY ${SRC_PATH}general/lib/gcp/* /usr/lib/python2.7/datalab/
-COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python2.7/datalab/common_lib.py
-COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python2.7/datalab/fab.py
+COPY ${SRC_PATH}general/lib/gcp/* /usr/lib/python3.8/datalab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
diff --git a/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile
index 0e5bf50..953b929 100644
--- a/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile
@@ -26,7 +26,7 @@
COPY dataengine-service/fabfile.py /root/
COPY dataengine-service/description.json /root/
COPY general/scripts/gcp/dataengine-service_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/common_* /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
diff --git a/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
index 60ba4c6..8445b20 100644
--- a/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
@@ -29,7 +29,7 @@
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/notebook_reconfigure_dataengine_spark.py /root/scripts/
COPY general/scripts/os/common_* /root/scripts/
COPY general/scripts/gcp/dataengine_* /root/scripts/
diff --git a/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile
index 7fb9053..45a327b 100644
--- a/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile
@@ -27,7 +27,7 @@
COPY deeplearning/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/deeplearning_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile
index 56e2aa1..ef3c1ac 100644
--- a/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile
@@ -26,7 +26,7 @@
COPY edge/ /root/
COPY general/scripts/gcp/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python2.7/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/files/gcp/ps_policy.json /root/files/ps_policy.json
COPY general/files/gcp/ps_roles.json /root/files/ps_roles.json
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile
index 1791d79..0de5dc7 100644
--- a/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile
@@ -27,7 +27,7 @@
COPY jupyter/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/jupyter_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile
index d9a2da2..4ab9fd2 100644
--- a/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile
@@ -27,7 +27,7 @@
COPY jupyterlab/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/jupyter* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/pyspark_local_template.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyterlab_description.json b/infrastructure-provisioning/src/general/files/gcp/jupyterlab_description.json
index 3f202f7..7615562 100644
--- a/infrastructure-provisioning/src/general/files/gcp/jupyterlab_description.json
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyterlab_description.json
@@ -29,5 +29,9 @@
"version": "jupyter_lab-0.35.6",
"vendor": "GCP"
}
+ ],
+ "gpu_types": [
+ {"Size": "S", "Gpu_type": "nvidia-tesla-t4"},
+ {"Size": "M", "Gpu_type": "nvidia-tesla-v100"}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile
index 502fcb2..8c8df3c 100644
--- a/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile
@@ -27,7 +27,7 @@
COPY project/ /root/
COPY general/scripts/gcp/project_* /root/scripts/
COPY general/scripts/gcp/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python2.7/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/files/gcp/ps_policy.json /root/files/ps_policy.json
COPY general/files/gcp/ps_roles.json /root/files/ps_roles.json
COPY general/templates/os/manage_step_certs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile
index c54a690..5738cba 100644
--- a/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile
@@ -27,7 +27,7 @@
COPY rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/gcp/core-site.xml /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile
index af553e4..a330912 100644
--- a/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile
@@ -25,7 +25,7 @@
COPY infrastructure-provisioning/src/ssn/ /root/
COPY infrastructure-provisioning/src/general/scripts/gcp/ssn_* /root/scripts/
-COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python2.7/datalab/ssn_lib.py
+COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY infrastructure-provisioning/src/general/files/gcp/ssn_policy.json /root/files/
COPY infrastructure-provisioning/src/general/files/gcp/ssn_roles.json /root/files/
COPY infrastructure-provisioning/src/general/templates/gcp/jenkins_jobs /root/templates/jenkins_jobs
diff --git a/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile
index 5423ef0..8c63e0a 100644
--- a/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile
@@ -27,7 +27,7 @@
COPY superset/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/superset_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/superset-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile
index 9f5c74c..0e85ff3 100644
--- a/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile
@@ -27,7 +27,7 @@
COPY tensor-rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/tensor-rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile
index 93ef83d..b585dd1 100644
--- a/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile
@@ -27,7 +27,7 @@
COPY tensor/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/tensor_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile
index c95dc6c..3ae5220 100644
--- a/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile
@@ -27,7 +27,7 @@
COPY zeppelin/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/zeppelin_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python2.7/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/gcp/interpreter_livy.json /root/templates/
COPY general/templates/gcp/interpreter_spark.json /root/templates/
COPY general/templates/os/dataengine_interpreter_livy.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 9b0e7d2..5f7e8ad 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -26,7 +26,6 @@
import datalab.fab
import json
import logging
-import meta_lib
import os
import sys
import time
@@ -1523,7 +1522,7 @@
def installing_python(region, bucket, user_name, cluster_name, application='', pip_mirror='', numpy_version='1.14.3'):
get_cluster_python_version(region, bucket, user_name, cluster_name)
- with file('/tmp/python_version') as f:
+ with open('/tmp/python_version') as f:
python_version = f.read()
python_version = python_version[0:5]
if not os.path.exists('/opt/python/python' + python_version):
@@ -1710,7 +1709,7 @@
zeppelin_restarted = False
default_port = 8998
get_cluster_python_version(region, bucket, user_name, cluster_name)
- with file('/tmp/python_version') as f:
+ with open('/tmp/python_version') as f:
python_version = f.read()
python_version = python_version[0:5]
livy_port = ''
diff --git a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
index e0a31c8..4f35072 100644
--- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
@@ -19,7 +19,7 @@
#
# ******************************************************************************
-import actions_lib
+import datalab.actions_lib
import backoff
import boto3
import json
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 9e9e3a8..efea3b4 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -24,9 +24,9 @@
import backoff
import datalab.common_lib
import datalab.fab
+import datalab.meta_lib
import json
import logging
-import meta_lib
import os
import sys
import time
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index d4aa77d..0d8d29b 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -23,16 +23,16 @@
import backoff
import datalab.common_lib
import datalab.fab
+import datalab.meta_lib
import google.auth
import json
import logging
-import meta_lib
import os
import random
import sys
import time
import traceback
-import urllib2
+import urllib3
from Crypto.PublicKey import RSA
from datalab.fab import *
from fabric.api import *
@@ -83,7 +83,7 @@
try:
print("Create VPC {}".format(vpc_name))
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'])
print("VPC {} has been created".format(vpc_name))
return result
except Exception as err:
@@ -98,7 +98,7 @@
request = self.service.networks().delete(project=self.project, network=vpc_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'])
print("VPC {} has been removed".format(vpc_name))
return result
except Exception as err:
@@ -121,7 +121,7 @@
try:
print("Create subnet {}".format(subnet_name))
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
print("Subnet {} has been created".format(subnet_name))
return result
except Exception as err:
@@ -136,7 +136,7 @@
request = self.service.subnetworks().delete(project=self.project, region=region, subnetwork=subnet_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
print("Subnet {} has been removed".format(subnet_name))
return result
except Exception as err:
@@ -151,7 +151,7 @@
request = self.service.firewalls().insert(project=self.project, body=firewall_params)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'])
print('Firewall {} created.'.format(firewall_params['name']))
return result
except Exception as err:
@@ -166,7 +166,7 @@
request = self.service.firewalls().delete(project=self.project, firewall=firewall_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'])
print('Firewall {} removed.'.format(firewall_name))
return result
except Exception as err:
@@ -181,7 +181,7 @@
request = self.service.routes().insert(project=self.project, body=nat_route_params)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'])
print('NAT route {} created.'.format(nat_route_params['name']))
return result
except Exception as err:
@@ -196,7 +196,7 @@
request = self.service.routes().delete(project=self.project, route=nat_route_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'])
print('NAT route {} deleteed.'.format(nat_route_name))
return result
except Exception as err:
@@ -280,7 +280,7 @@
"sourceImage": secondary_image_name}
request = self.service.disks().insert(project=self.project, zone=zone, body=params)
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
print('Disk {}-secondary created.'.format(instance_name))
return request
except Exception as err:
@@ -296,7 +296,7 @@
request = self.service.disks().delete(project=self.project, zone=zone, disk=instance_name + '-secondary')
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
print('Disk {}-secondary removed.'.format(instance_name))
except errors.HttpError as err:
if err.resp.status == 404:
@@ -320,8 +320,8 @@
primary_disk_size='12', secondary_disk_size='30',
gpu_accelerator_type='None'):
key = RSA.importKey(open(ssh_key_path, 'rb').read())
- ssh_key = key.publickey().exportKey("OpenSSH")
- unique_index = meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
+ ssh_key = key.publickey().exportKey("OpenSSH").decode('UTF-8')
+ unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
service_account_email = "{}-{}@{}.iam.gserviceaccount.com".format(service_base_name, unique_index, self.project)
access_configs = ''
if instance_class == 'edge':
@@ -455,7 +455,7 @@
body=instance_params)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
print('Instance {} created.'.format(instance_name))
request = self.service.instances().get(instance=instance_name, project=self.project,
zone=zone)
@@ -510,7 +510,7 @@
instance=instance_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
print('Instance {} removed.'.format(instance_name))
return result
except Exception as err:
@@ -525,7 +525,7 @@
request = self.service.instances().stop(project=self.project, zone=zone, instance=instance_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
return True
except Exception as err:
logging.info(
@@ -539,7 +539,7 @@
request = self.service.instances().start(project=self.project, zone=zone, instance=instance_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], zone=zone)
return True
except Exception as err:
logging.info(
@@ -550,16 +550,16 @@
traceback.print_exc(file=sys.stdout)
def remove_service_account(self, service_account_name, service_base_name):
- unique_index = meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
+ unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
service_account_email = "{}-{}@{}.iam.gserviceaccount.com".format(service_base_name, unique_index, self.project)
request = self.service_iam.projects().serviceAccounts().delete(
name='projects/{}/serviceAccounts/{}'.format(self.project, service_account_email))
try:
result = request.execute()
- service_account_removed = meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
+ service_account_removed = datalab.meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
while service_account_removed:
time.sleep(5)
- service_account_removed = meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
+ service_account_removed = datalab.meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
time.sleep(30)
print('Service account {} removed.'.format(service_account_name))
return result
@@ -580,10 +580,10 @@
body=params)
try:
result = request.execute()
- service_account_created = meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
+ service_account_created = datalab.meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
while not service_account_created:
time.sleep(5)
- service_account_created = meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
+ service_account_created = datalab.meta_lib.GCPMeta().get_service_account(service_account_name, service_base_name)
time.sleep(30)
print('Service account {} created.'.format(service_account_name))
return result
@@ -601,7 +601,7 @@
num += 1
request = GCPActions().service_resource.projects().getIamPolicy(resource=self.project, body={})
project_policy = request.execute()
- unique_index = meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
+ unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
service_account_email = "{}-{}@{}.iam.gserviceaccount.com".format(service_base_name, unique_index, self.project)
params = {
"role": "projects/{}/roles/{}".format(self.project, role_name.replace('-', '_')),
@@ -644,10 +644,10 @@
}})
try:
result = request.execute()
- role_created = meta_lib.GCPMeta().get_role(role_name)
+ role_created = datalab.meta_lib.GCPMeta().get_role(role_name)
while not role_created:
time.sleep(5)
- role_created = meta_lib.GCPMeta().get_role(role_name)
+ role_created = datalab.meta_lib.GCPMeta().get_role(role_name)
time.sleep(30)
print('IAM role {} created.'.format(role_name))
return result
@@ -667,14 +667,14 @@
{ })
try:
result = request.execute()
- role = meta_lib.GCPMeta().get_role(role_name)
+ role = datalab.meta_lib.GCPMeta().get_role(role_name)
if 'deleted' in role:
role_removed = True
else:
role_removed = False
while role_removed:
time.sleep(5)
- role = meta_lib.GCPMeta().get_role(role_name)
+ role = datalab.meta_lib.GCPMeta().get_role(role_name)
if 'deleted' in role:
role_removed = True
time.sleep(30)
@@ -694,14 +694,14 @@
name='projects/{}/roles/{}'.format(self.project, role_name.replace('-', '_')))
try:
result = request.execute()
- role = meta_lib.GCPMeta().get_role(role_name)
+ role = datalab.meta_lib.GCPMeta().get_role(role_name)
if 'deleted' in role:
role_removed = True
else:
role_removed = False
while not role_removed:
time.sleep(5)
- role = meta_lib.GCPMeta().get_role(role_name)
+ role = datalab.meta_lib.GCPMeta().get_role(role_name)
if 'deleted' in role:
role_removed = True
time.sleep(30)
@@ -739,7 +739,7 @@
traceback.print_exc(file=sys.stdout)
def set_service_account_to_instance(self, service_account_name, instance_name, service_base_name):
- unique_index = meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
+ unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
service_account_email = "{}-{}@{}.iam.gserviceaccount.com".format(service_base_name, unique_index, self.project)
params = {
"email": service_account_email
@@ -762,7 +762,7 @@
request = self.service.addresses().insert(project=self.project, region=region, body=params)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
print('Static address {} created.'.format(address_name))
return result
except Exception as err:
@@ -778,7 +778,7 @@
request = self.service.addresses().delete(project=self.project, region=region, address=address_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'], region=region)
print('Static address {} removed.'.format(address_name))
return result
except Exception as err:
@@ -802,16 +802,16 @@
id_list=[]
try:
GCPActions().stop_instance(instance_name, zone)
- primary_image_check = meta_lib.GCPMeta().get_image_by_name(primary_image_name)
+ primary_image_check = datalab.meta_lib.GCPMeta().get_image_by_name(primary_image_name)
if primary_image_check != '':
GCPActions().start_instance(instance_name, zone)
return ''
primary_result = primary_request.execute()
secondary_result = secondary_request.execute()
- meta_lib.GCPMeta().wait_for_operation(primary_result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(primary_result['name'])
print('Image {} has been created.'.format(primary_image_name))
id_list.append(primary_result.get('id'))
- meta_lib.GCPMeta().wait_for_operation(secondary_result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(secondary_result['name'])
print('Image {} has been created.'.format(secondary_image_name))
id_list.append(secondary_result.get('id'))
GCPActions().start_instance(instance_name, zone)
@@ -831,7 +831,7 @@
request = self.service.images().delete(project=self.project, image=image_name)
try:
result = request.execute()
- meta_lib.GCPMeta().wait_for_operation(result['name'])
+ datalab.meta_lib.GCPMeta().wait_for_operation(result['name'])
print('Image {} was removed.'.format(image_name))
except errors.HttpError as err:
if err.resp.status == 404:
@@ -870,7 +870,7 @@
def set_bucket_owner(self, bucket_name, service_account_name, service_base_name):
try:
- unique_index = meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
+ unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
service_account_email = "{}-{}@{}.iam.gserviceaccount.com".format(service_base_name, unique_index,
self.project)
bucket = self.storage_client.get_bucket(bucket_name)
@@ -919,11 +919,11 @@
try:
result = request.execute()
time.sleep(5)
- cluster_status = meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
+ cluster_status = datalab.meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
while cluster_status[0]['status'] != 'running':
time.sleep(5)
print('The cluster is being created... Please wait')
- cluster_status = meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
+ cluster_status = datalab.meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
if cluster_status[0]['status'] == 'terminated':
raise Exception
return result
@@ -979,11 +979,11 @@
request = self.dataproc.projects().regions().clusters().delete(projectId=self.project, region=region, clusterName=cluster_name)
try:
result = request.execute()
- cluster_status = meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
+ cluster_status = datalab.meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
while cluster_status[0]['status'] != 'terminated':
time.sleep(5)
print('The cluster is being terminated... Please wait')
- cluster_status = meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
+ cluster_status = datalab.meta_lib.GCPMeta().get_list_cluster_statuses([cluster_name])
GCPActions().delete_dataproc_jobs(cluster_name)
return result
except Exception as err:
@@ -1027,10 +1027,10 @@
try:
res = request.execute()
print("Job ID: {}".format(res['reference']['jobId']))
- job_status = meta_lib.GCPMeta().get_dataproc_job_status(res['reference']['jobId'])
+ job_status = datalab.meta_lib.GCPMeta().get_dataproc_job_status(res['reference']['jobId'])
while job_status != 'done':
time.sleep(5)
- job_status = meta_lib.GCPMeta().get_dataproc_job_status(res['reference']['jobId'])
+ job_status = datalab.meta_lib.GCPMeta().get_dataproc_job_status(res['reference']['jobId'])
if job_status in ('failed', 'error'):
raise Exception
return job_status
@@ -1047,7 +1047,7 @@
def delete_dataproc_jobs(self, cluster_filter):
try:
- jobs = meta_lib.GCPMeta().get_dataproc_jobs()
+ jobs = datalab.meta_lib.GCPMeta().get_dataproc_jobs()
cluster_jobs_ids = [job['reference']['jobId'] for job in jobs
if cluster_filter in job['placement']['clusterName']]
for job_id in list(set(cluster_jobs_ids)):
@@ -1075,7 +1075,7 @@
try:
version_file = '{0}/{1}/{2}_version'.format(user_name, cluster_name, application)
if GCPActions().get_from_bucket(bucket, version_file, '/tmp/{}_version'.format(application)):
- with file('/tmp/{}_version'.format(application)) as f:
+ with open('/tmp/{}_version'.format(application)) as f:
version = f.read()
return version[0:5]
else:
@@ -1140,7 +1140,7 @@
def remove_kernels(self, notebook_name, dataproc_name, dataproc_version, ssh_user, key_path, computational_name):
try:
- notebook_ip = meta_lib.GCPMeta().get_private_ip_address(notebook_name)
+ notebook_ip = datalab.meta_lib.GCPMeta().get_private_ip_address(notebook_name)
env.hosts = "{}".format(notebook_ip)
env.user = "{}".format(ssh_user)
env.key_filename = "{}".format(key_path)
@@ -1205,7 +1205,7 @@
zeppelin_restarted = False
default_port = 8998
GCPActions().get_cluster_app_version(bucket, user_name, cluster_name, 'python')
- with file('/tmp/python_version') as f:
+ with open('/tmp/python_version') as f:
python_version = f.read()
python_version = python_version[0:5]
livy_port = ''
@@ -1296,7 +1296,7 @@
def install_python(self, bucket, user_name, cluster_name, application, numpy_version='1.14.3'):
try:
GCPActions().get_cluster_app_version(bucket, user_name, cluster_name, 'python')
- with file('/tmp/python_version') as f:
+ with open('/tmp/python_version') as f:
python_version = f.read()
python_version = python_version[0:5]
if not os.path.exists('/opt/python/python{}'.format(python_version)):
@@ -1449,7 +1449,7 @@
def remove_dataengine_kernels(notebook_name, os_user, key_path, cluster_name):
try:
computational_name = os.environ['computational_name'].replace('_', '-').lower()
- private = meta_lib.get_instance_private_ip_address(cluster_name, notebook_name)
+ private = datalab.meta_lib.get_instance_private_ip_address(cluster_name, notebook_name)
env.hosts = "{}".format(private)
env.user = "{}".format(os_user)
env.key_filename = "{}".format(key_path)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
index 6fcb8ab..0a11b58 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -28,7 +28,7 @@
import time
-def manage_pkg(command, environment, requisites, warn='False'):
+def manage_pkg(command, environment, requisites):
try:
allow = False
counter = 0
@@ -39,28 +39,50 @@
else:
print('Package manager is:')
if environment == 'remote':
- if sudo('pgrep "^apt" -a && echo "busy" || echo "ready"') == 'busy':
+ if sudo('pgrep "^apt" -a && echo "busy" || echo "ready"') == 'busy' or sudo('pgrep "^dpkg" -a && echo "busy" || echo "ready"') == 'busy':
counter += 1
time.sleep(10)
else:
try:
- sudo('sudo dpkg --configure -a', warn_only=warn)
- except:
- PID = sudo('lsof /var/lib/dpkg/lock-frontend | grep dpkg | awk \'{print $2}\'')
- sudo('kill -9 {}'.format(PID))
- sudo('rm -f /var/lib/dpkg/lock-frontend')
- sudo('sudo dpkg --configure -a')
- sudo('sudo apt update')
- try:
- sudo('apt-get {0} {1}'.format(command, requisites), warn_only=warn)
- except:
- sudo('lsof /var/lib/dpkg/lock')
- sudo('lsof /var/lib/apt/lists/lock')
- sudo('lsof /var/cache/apt/archives/lock')
- sudo('rm -f /var/lib/apt/lists/lock')
- sudo('rm -f /var/cache/apt/archives/lock')
- sudo('rm -f /var/lib/dpkg/lock')
- allow = True
+ error_parser = "frontend is locked|locked"
+ sudo('dpkg --configure -a 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({0})" /tmp/tee.tmp > '
+ '/tmp/dpkg.log; then echo "" > /tmp/dpkg.log;fi'.format(error_parser))
+ err = sudo('cat /tmp/dpkg.log')
+ count = 0
+ while err != '' and count < 10:
+ pid = sudo('lsof /var/lib/dpkg/lock-frontend | grep dpkg | awk \'{print $2}\'')
+ if pid != '':
+ sudo('kill -9 {}'.format(pid))
+ sudo('rm -f /var/lib/dpkg/lock-frontend')
+ pid = sudo('lsof /var/lib/dpkg/lock | grep dpkg | awk \'{print $2}\'')
+ elif pid != '':
+ sudo('kill -9 {}'.format(pid))
+ sudo('rm -f /var/lib/dpkg/lock')
+ sudo('dpkg --configure -a 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({0})" /tmp/tee.tmp > '
+ '/tmp/dpkg.log; then echo "" > /tmp/dpkg.log;fi'.format(error_parser))
+ err = sudo('cat /tmp/dpkg.log')
+ count = count + 1
+ sudo('apt update')
+
+ sudo('apt-get {0} {1} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > '
+ '/tmp/apt.log; then echo "" > /tmp/apt.log;fi'.format(command, requisites, error_parser))
+ err = sudo('cat /tmp/apt.log')
+ count = 0
+ while err != '' and count < 10:
+ sudo('lsof /var/lib/dpkg/lock')
+ sudo('lsof /var/lib/apt/lists/lock')
+ sudo('lsof /var/cache/apt/archives/lock')
+ sudo('rm -f /var/lib/apt/lists/lock')
+ sudo('rm -f /var/cache/apt/archives/lock')
+ sudo('rm -f /var/lib/dpkg/lock')
+ sudo('apt-get {0} {1} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > '
+ '/tmp/apt.log; then echo "" > /tmp/apt.log;fi'.format(command, requisites, error_parser))
+ err = sudo('cat /tmp/apt.log')
+ count = count + 1
+ allow = True
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to manage_pkgs", str(err))
elif environment == 'local':
if local('sudo pgrep "^apt" -a && echo "busy" || echo "ready"', capture=True) == 'busy':
counter += 1
@@ -73,7 +95,7 @@
except:
sys.exit(1)
-def ensure_pkg(user, requisites='linux-headers-generic python-pip python-dev '
+def ensure_pkg(user, requisites='linux-headers-generic python3-pip python3-dev '
'groff gcc vim less git wget '
'libssl-dev unattended-upgrades nmap '
'libffi-dev unzip libxml2-dev haveged'):
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index d00454a..1e727e7 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 52a4359..72f0fbd 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -172,12 +172,10 @@
try:
sudo("sudo sed -i~orig -e 's/# deb-src/deb-src/' /etc/apt/sources.list")
manage_pkg('update', 'remote', '')
- manage_pkg('-y build-dep', 'remote', 'python-matplotlib')
- sudo('pip2 install matplotlib==2.0.2 --no-cache-dir')
+ manage_pkg('-y build-dep', 'remote', 'python3-matplotlib')
sudo('pip3 install matplotlib==2.0.2 --no-cache-dir')
if os.environ['application'] in ('tensor', 'deeplearning'):
- sudo('python2.7 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy2_version']))
- sudo('python3.6 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
+ sudo('python3.8 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('touch /home/' + os_user + '/.ensure_dir/matplot_ensured')
except:
sys.exit(1)
@@ -228,10 +226,8 @@
try:
manage_pkg('-y install', 'remote', 'libjpeg8-dev zlib1g-dev')
if os.environ['application'] in ('jupyter', 'zeppelin'):
- sudo('pip2 install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'.format(os.environ['notebook_numpy2_version']))
sudo('pip3 install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'.format(os.environ['notebook_numpy_version']))
if os.environ['application'] in ('tensor', 'deeplearning'):
- sudo('pip2 install opencv-python==4.2.0.32 h5py --no-cache-dir')
sudo('pip3 install opencv-python h5py --no-cache-dir')
sudo('touch /home/' + os_user + '/.ensure_dir/additional_python_libs_ensured')
except:
@@ -249,35 +245,13 @@
except:
sys.exit(1)
-
-def ensure_python2_libraries(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/python2_libraries_ensured'):
- try:
- try:
- manage_pkg('-y install', 'remote', 'libssl1.0-dev python-virtualenv')
- except:
- sudo('pip2 install virtualenv --no-cache-dir')
- manage_pkg('-y install', 'remote', 'libssl1.0-dev')
- try:
- sudo('pip2 install tornado=={0} ipython ipykernel=={1} --no-cache-dir' \
- .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
- except:
- sudo('pip2 install tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \
- .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
- sudo('pip2 install -U pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
- sudo('pip2 install boto3 backoff --no-cache-dir')
- sudo('pip2 install fabvenv fabric-virtualenv future --no-cache-dir')
- sudo('touch /home/' + os_user + '/.ensure_dir/python2_libraries_ensured')
- except:
- sys.exit(1)
-
-
def ensure_python3_libraries(os_user):
if not exists('/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'):
try:
#manage_pkg('-y install', 'remote', 'python3-setuptools')
manage_pkg('-y install', 'remote', 'python3-pip')
manage_pkg('-y install', 'remote', 'libkrb5-dev')
+ sudo('pip3 install -U keyrings.alt backoff')
sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
try:
sudo('pip3 install tornado=={0} ipython==7.9.0 ipykernel=={1} sparkmagic --no-cache-dir' \
@@ -338,9 +312,7 @@
'echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc')
# install TensorFlow and run TensorBoard
# sudo('python2.7 -m pip install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
- sudo('python3 -m pip install --upgrade '
- 'https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp36-cp36m-manylinux2010_x86_64.whl'
- ' --no-cache-dir'.format(tensorflow_version))
+ sudo('python3 -m pip install --upgrade tensorflow-gpu=={} --no-cache-dir'.format(tensorflow_version))
sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'.format(os_user))
put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service')
sudo("sed -i 's|OS_USR|{}|' /tmp/tensorboard.service".format(os_user))
@@ -376,7 +348,6 @@
def install_livy_dependencies(os_user):
if not exists('/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
manage_pkg('-y install', 'remote', 'libkrb5-dev')
- sudo('pip2 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
sudo('pip3 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
sudo('touch /home/' + os_user + '/.ensure_dir/livy_dependencies_ensured')
@@ -390,7 +361,6 @@
def install_livy_dependencies_emr(os_user):
if not os.path.exists('/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
manage_pkg('-y install', 'local', 'libkrb5-dev')
- local('sudo pip2 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
local('sudo pip3 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
local('touch /home/' + os_user + '/.ensure_dir/livy_dependencies_ensured')
@@ -495,14 +465,11 @@
env.shell = "/bin/bash -l -c -i"
manage_pkg('update', 'remote', '')
manage_pkg('-y install --no-install-recommends', 'remote', 'build-essential cmake git libgoogle-glog-dev '
- 'libprotobuf-dev protobuf-compiler python-dev python-pip')
- sudo('pip2 install numpy=={} protobuf --no-cache-dir'.format(os.environ['notebook_numpy2_version']))
+ 'libprotobuf-dev protobuf-compiler python3-dev python3-pip')
sudo('pip3 install numpy=={} protobuf --no-cache-dir'.format(os.environ['notebook_numpy_version']))
manage_pkg('-y install --no-install-recommends', 'remote', 'libgflags-dev')
manage_pkg('-y install --no-install-recommends', 'remote', 'libgtest-dev libiomp-dev libleveldb-dev liblmdb-dev '
'libopencv-dev libopenmpi-dev libsnappy-dev openmpi-bin openmpi-doc python-pydot')
- sudo('pip2 install flask graphviz hypothesis jupyter matplotlib==2.0.2 pydot python-nvd3 pyyaml requests scikit-image '
- 'scipy setuptools tornado --no-cache-dir')
sudo('pip3 install flask graphviz hypothesis jupyter matplotlib==2.0.2 pydot python-nvd3 pyyaml requests scikit-image '
'scipy setuptools tornado --no-cache-dir')
sudo('cp -f /opt/cudnn/include/* /opt/cuda-{}/include/'.format(os.environ['notebook_cuda_version']))
@@ -523,30 +490,26 @@
sudo('touch /home/' + os_user + '/.ensure_dir/caffe2_ensured')
-def install_cntk(os_user, cntk2_version, cntk_version):
+def install_cntk(os_user, cntk_version):
if not exists('/home/{}/.ensure_dir/cntk_ensured'.format(os_user)):
- sudo('pip2 install cntk=={} --no-cache-dir'.format(cntk2_version))
sudo('pip3 install cntk-gpu=={} --no-cache-dir'.format(cntk_version))
sudo('touch /home/{}/.ensure_dir/cntk_ensured'.format(os_user))
def install_keras(os_user, keras_version):
if not exists('/home/{}/.ensure_dir/keras_ensured'.format(os_user)):
- sudo('pip2 install keras=={} --no-cache-dir'.format(keras_version))
sudo('pip3 install keras=={} --no-cache-dir'.format(keras_version))
sudo('touch /home/{}/.ensure_dir/keras_ensured'.format(os_user))
def install_theano(os_user, theano_version):
if not exists('/home/{}/.ensure_dir/theano_ensured'.format(os_user)):
- sudo('python2.7 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
sudo('python3 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
sudo('touch /home/{}/.ensure_dir/theano_ensured'.format(os_user))
def install_mxnet(os_user, mxnet_version):
if not exists('/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
- sudo('pip2 install mxnet-cu101=={} opencv-python==4.2.0.32 --no-cache-dir'.format(mxnet_version))
sudo('pip3 install mxnet-cu101=={} opencv-python --no-cache-dir'.format(mxnet_version))
sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index ee53977..7ff0968 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -41,7 +41,7 @@
stable"')
manage_pkg('update', 'remote', '')
sudo('apt-cache policy docker-ce')
- manage_pkg('-y install', 'remote', 'docker-ce={}~ce~3-0~ubuntu'.format(docker_version))
+ manage_pkg('-y install', 'remote', 'docker-ce=5:{}~3-0~ubuntu-focal'.format(docker_version))
sudo('usermod -a -G docker ' + os_user)
sudo('update-rc.d docker defaults')
sudo('update-rc.d docker enable')
@@ -331,7 +331,7 @@
keycloak_client_id,
keycloak_client_secret,
keycloak_auth_server_url)
- sudo('python /tmp/configure_billing.py {}'.format(params))
+ sudo('python3 /tmp/configure_billing.py {}'.format(params))
try:
if os.environ['conf_stepcerts_enabled'] == 'true':
sudo(
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 0d99831..ea42c4e 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -42,9 +42,9 @@
if not exists('/home/{}/.ensure_dir/pip_path_added'.format(os.environ['conf_os_user'])):
sudo('echo PATH=$PATH:/usr/local/bin/:/opt/spark/bin/ >> /etc/profile')
sudo('echo export PATH >> /etc/profile')
- sudo('pip install -UI pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
- sudo('pip install --upgrade setuptools')
- sudo('pip install -U {} --no-cache-dir'.format(requisites))
+ sudo('pip3 install -UI pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+ sudo('pip3 install --upgrade setuptools')
+ sudo('pip3 install -UI {} --no-cache-dir'.format(requisites))
sudo('touch /home/{}/.ensure_dir/pip_path_added'.format(os.environ['conf_os_user']))
except:
sys.exit(1)
@@ -186,15 +186,12 @@
keyfile = os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem"
env.key_filename = [keyfile]
env.host_string = os_user + '@' + hostname
- sudo('python ' + datalab_path + 'tmp/resource_status.py --resource {} --status {}'.format(resource, status))
+ sudo('python3 ' + datalab_path + 'tmp/resource_status.py --resource {} --status {}'.format(resource, status))
def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version, exploratory_name):
if not exists('/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
- sudo('pip2 install pyrsistent==0.16.0 --no-cache-dir')
- sudo('pip2 install notebook==5.7.8 --no-cache-dir')
- sudo('pip2 install jupyter --no-cache-dir')
sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
sudo('pip3 install jupyter --no-cache-dir')
sudo('rm -rf {}'.format(jupyter_conf_file))
@@ -258,7 +255,7 @@
stable"')
manage_pkg('update', 'remote', '')
sudo('apt-cache policy docker-ce')
- manage_pkg('-y install', 'remote', 'docker-ce={}~ce~3-0~ubuntu'.format(docker_version), 'True')
+ manage_pkg('-y install', 'remote', 'docker-ce=5:{}~3-0~ubuntu-focal'.format(docker_version))
sudo('touch /home/{}/.ensure_dir/docker_ensured'.format(os_user))
except Exception as err:
print('Failed to configure Docker:', str(err))
@@ -387,9 +384,9 @@
text = text.replace('CLUSTER_NAME', cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '2.7')
- text = text.replace('PYTHON_FULL_VERSION', '2.7')
- text = text.replace('PYTHON_PATH', '/usr/bin/python2.7')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
text = text.replace('DATAENGINE-SERVICE_VERSION', dataengine_service_version)
with open(kernel_path, 'w') as f:
f.write(text)
@@ -398,7 +395,7 @@
format(dataengine_service_version, cluster_name, kernel_path, os_user))
local('sudo mv /tmp/kernel_var.json ' + kernel_path)
get_cluster_python_version(region, bucket, user_name, cluster_name)
- with file('/tmp/python_version') as f:
+ with open('/tmp/python_version') as f:
python_version = f.read()
if python_version != '\n':
installing_python(region, bucket, user_name, cluster_name, application, pip_mirror, numpy_version)
@@ -571,12 +568,13 @@
r_pkgs = dict()
sudo('R -e \'write.table(available.packages(contriburl="https://cloud.r-project.org/src/contrib"), file="/tmp/r.csv", row.names=F, col.names=F, sep=",")\'')
get("/tmp/r.csv", "r.csv")
- with open('r.csv', 'rb') as csvfile:
+ with open('r.csv', 'r') as csvfile:
reader = csv.reader(csvfile, delimiter=',')
for row in reader:
r_pkgs[row[0]] = row[1]
return r_pkgs
- except:
+ except Exception as err:
+ print("Failed to install {} ".format(err))
sys.exit(1)
@@ -611,7 +609,7 @@
run('git config --global user.name "Example User"')
run('git config --global user.email "example@example.com"')
run('mkdir -p ~/.git/templates/hooks')
- put('/root/scripts/git_pre_commit.py', '~/.git/templates/hooks/pre-commit', mode=0755)
+ put('/root/scripts/git_pre_commit.py', '~/.git/templates/hooks/pre-commit', mode=0o755)
run('git config --global init.templatedir ~/.git/templates')
run('touch ~/.gitignore')
run('git config --global core.excludesfile ~/.gitignore')
@@ -730,11 +728,6 @@
env['connection_attempts'] = 100
env.key_filename = [keyfile]
env.host_string = os_user + '@' + hostname
- if not exists('/usr/bin/pip2'):
- if not exists('/usr/bin/pip-2.7'):
- manage_pkg('-y install', 'remote', 'python2-pip')
- else:
- sudo('ln -s /usr/bin/pip-2.7 /usr/bin/pip2')
manage_pkg('-y install', 'remote', 'python3-pip')
if not exists('/usr/bin/pip3') and sudo("python3.4 -V 2>/dev/null | awk '{print $2}'"):
sudo('ln -s /usr/bin/pip-3.4 /usr/bin/pip3')
@@ -744,9 +737,10 @@
sudo('ln -s /usr/bin/pip-3.6 /usr/bin/pip3')
elif not exists('/usr/bin/pip3') and sudo("python3.7 -V 2>/dev/null | awk '{print $2}'"):
sudo('ln -s /usr/bin/pip-3.7 /usr/bin/pip3')
+ elif not exists('/usr/bin/pip3') and sudo("python3.8 -V 2>/dev/null | awk '{print $2}'"):
+ sudo('ln -s /usr/bin/pip-3.8 /usr/bin/pip3')
if emr:
sudo('pip3 install -U pip=={}'.format(os.environ['conf_pip_version']))
- sudo('pip2 install -U pip=={}'.format(os.environ['conf_pip_version']))
sudo('ln -s /usr/local/bin/pip3.7 /bin/pip3.7')
sudo('echo "export PATH=$PATH:/usr/local/bin" >> /etc/profile')
sudo('source /etc/profile')
@@ -855,7 +849,6 @@
try:
if exists('/usr/bin/pip3'):
sudo('pip3 install -U pyopenssl')
- sudo('pip2 install -U pyopenssl')
sudo('touch /home/{}/.ensure_dir/pyopenssl_updated'.format(os_user))
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index 7047633..6343635 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -78,7 +78,7 @@
sudo('wget --no-check-certificate https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm')
sudo('rpm -ivh epel-release-latest-7.noarch.rpm')
manage_pkg('repolist', 'remote', '')
- manage_pkg('-y install', 'remote', 'python-pip gcc')
+ manage_pkg('-y install', 'remote', 'python3-pip gcc')
sudo('rm -f epel-release-latest-7.noarch.rpm')
sudo('export LC_ALL=C')
manage_pkg('-y install', 'remote', requisites)
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index 27f346c..948b3fa 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index 284c941..50e2193 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -48,7 +48,7 @@
def downgrade_python_version():
try:
- sudo('python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); \
+ sudo('python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); \
version = [pkg.vr for pkg in pl.installed if pkg.name == \'python\']; \
os.system(\'yum -y downgrade python python-devel-2.7.5-58.el7.x86_64 python-libs-2.7.5-58.el7.x86_64\') \
if version != [] and version[0] == \'2.7.5-68.el7\' else False"')
@@ -142,11 +142,9 @@
def ensure_matplot(os_user):
if not exists('/home/{}/.ensure_dir/matplot_ensured'.format(os_user)):
try:
- sudo('pip2 install matplotlib==2.0.2 --no-cache-dir')
sudo('python3.5 -m pip install matplotlib==2.0.2 --no-cache-dir')
if os.environ['application'] in ('tensor', 'deeplearning'):
- sudo('python2.7 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
- sudo('python3.5 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
+ sudo('python3.8 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('touch /home/{}/.ensure_dir/matplot_ensured'.format(os_user))
except:
sys.exit(1)
@@ -188,11 +186,9 @@
manage_pkg('clean', 'remote', 'all')
manage_pkg('-y install', 'remote', 'zlib-devel libjpeg-turbo-devel --nogpgcheck')
if os.environ['application'] in ('jupyter', 'zeppelin'):
- sudo('pip2 install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('python3.5 -m pip install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'.format(os.environ['notebook_numpy_version']))
if os.environ['application'] in ('tensor', 'deeplearning'):
- sudo('python2.7 -m pip install opencv-python==4.2.0.32 h5py --no-cache-dir')
- sudo('python3.5 -m pip install opencv-python h5py --no-cache-dir')
+ sudo('python3.8 -m pip install opencv-python h5py --no-cache-dir')
sudo('touch /home/' + os_user + '/.ensure_dir/additional_python_libs_ensured')
except:
sys.exit(1)
@@ -211,30 +207,6 @@
except:
sys.exit(1)
-
-def ensure_python2_libraries(os_user):
- if not exists('/home/' + os_user + '/.ensure_dir/python2_libraries_ensured'):
- try:
- sudo('pip2 install pyparsing==2.0.3')
- manage_pkg('-y install', 'remote', 'python-setuptools python-wheel')
- manage_pkg('-y install', 'remote', 'python-virtualenv openssl-devel python-devel openssl-libs libxslt-devel --nogpgcheck')
- try:
- sudo('python2 -m pip install backports.shutil_get_terminal_size tornado=={0} ipython ipykernel=={1} --no-cache-dir' \
- .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
- except:
- sudo('python2 -m pip install backports.shutil_get_terminal_size tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \
- .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
- sudo('echo y | python2 -m pip uninstall backports.shutil_get_terminal_size')
- sudo('python2 -m pip install backports.shutil_get_terminal_size --no-cache-dir')
- sudo('pip2 install -UI pip=={} setuptools --no-cache-dir'.format(os.environ['conf_pip_version']))
- sudo('pip2 install boto3 backoff --no-cache-dir')
- sudo('pip2 install fabvenv fabric-virtualenv future --no-cache-dir')
- downgrade_python_version()
- sudo('touch /home/' + os_user + '/.ensure_dir/python2_libraries_ensured')
- except:
- sys.exit(1)
-
-
def ensure_python3_libraries(os_user):
if not exists('/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'):
try:
@@ -288,8 +260,7 @@
# install TensorFlow and run TensorBoard
sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl'.format(tensorflow_version))
sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl'.format(tensorflow_version))
- sudo('python2.7 -m pip install --upgrade tensorflow_gpu-{}-cp27-none-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
- sudo('python3.5 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
+ sudo('python3.8 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
sudo('rm -rf /home/{}/tensorflow_gpu-*'.format(os_user))
sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'.format(os_user))
put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service')
@@ -314,7 +285,6 @@
def install_livy_dependencies(os_user):
if not exists('/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
- sudo('pip2 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
sudo('pip3.5 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
sudo('touch /home/' + os_user + '/.ensure_dir/livy_dependencies_ensured')
@@ -329,7 +299,6 @@
def install_livy_dependencies_emr(os_user):
if not os.path.exists('/home/' + os_user + '/.ensure_dir/livy_dependencies_ensured'):
- local('sudo -i pip2 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
local('sudo -i pip3.5 install cloudpickle requests requests-kerberos flake8 flaky pytest --no-cache-dir')
local('touch /home/' + os_user + '/.ensure_dir/livy_dependencies_ensured')
@@ -375,7 +344,7 @@
dep = [i for i in dep if i]
versions = []
res = sudo(
- 'python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\']"'.format(
+ 'python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\']"'.format(
name)).split('\r\n')[1]
if err:
status_msg = 'installation_error'
@@ -417,7 +386,7 @@
try:
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
downgrade_python_version()
- yum_raw = sudo('python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print {pkg.name:pkg.vr for pkg in pl.available}"')
+ yum_raw = sudo('python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print {pkg.name:pkg.vr for pkg in pl.available}"')
yum_re = re.sub\
(r'\w*\s\w*\D\s\w*.\w*.\s\w*.\w*.\w.\w*.\w*.\w*', '', yum_raw)
yum_list = yum_re.replace("'", "\"")
@@ -431,7 +400,6 @@
def install_opencv(os_user):
if not exists('/home/{}/.ensure_dir/opencv_ensured'.format(os_user)):
manage_pkg('-y install', 'remote', 'cmake python34 python34-devel python34-pip gcc gcc-c++')
- sudo('pip2 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('pip3.4 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('pip3.5 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
run('git clone https://github.com/opencv/opencv.git')
@@ -450,8 +418,6 @@
env.shell = "/bin/bash -l -c -i"
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install --nogpgcheck', 'remote', 'automake cmake3 gcc gcc-c++ kernel-devel leveldb-devel lmdb-devel libtool protobuf-devel graphviz')
- sudo('pip2 install flask graphviz hypothesis jupyter matplotlib==2.0.2 numpy=={} protobuf pydot python-nvd3 pyyaml '
- 'requests scikit-image scipy setuptools tornado future --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('pip3.5 install flask graphviz hypothesis jupyter matplotlib==2.0.2 numpy=={} protobuf pydot python-nvd3 pyyaml '
'requests scikit-image scipy setuptools tornado future --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('cp /opt/cudnn/include/* /opt/cuda-8.0/include/')
@@ -478,28 +444,24 @@
manage_pkg('clean', 'remote', 'all')
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install --nogpgcheck', 'remote', 'openmpi openmpi-devel')
- sudo('pip2 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp27-cp27mu-linux_x86_64.whl --no-cache-dir'.format(cntk_version))
sudo('pip3.5 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(cntk_version))
sudo('touch /home/{}/.ensure_dir/cntk_ensured'.format(os_user))
def install_keras(os_user, keras_version):
if not exists('/home/{}/.ensure_dir/keras_ensured'.format(os_user)):
- sudo('pip2 install keras=={} --no-cache-dir'.format(keras_version))
sudo('pip3.5 install keras=={} --no-cache-dir'.format(keras_version))
sudo('touch /home/{}/.ensure_dir/keras_ensured'.format(os_user))
def install_theano(os_user, theano_version):
if not exists('/home/{}/.ensure_dir/theano_ensured'.format(os_user)):
- sudo('python2.7 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
- sudo('python3.5 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
+ sudo('python3.8 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
sudo('touch /home/{}/.ensure_dir/theano_ensured'.format(os_user))
def install_mxnet(os_user, mxnet_version):
if not exists('/home/{}/.ensure_dir/mxnet_ensured'.format(os_user)):
- sudo('pip2 install mxnet-cu80=={} opencv-python==4.2.0.32 --no-cache-dir'.format(mxnet_version))
sudo('pip3.5 install mxnet-cu80=={} opencv-python --no-cache-dir'.format(mxnet_version))
sudo('touch /home/{}/.ensure_dir/mxnet_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index 8a2c357..909e7b8 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -344,7 +344,7 @@
keycloak_client_id,
keycloak_client_secret,
keycloak_auth_server_url)
- sudo('python /tmp/configure_billing.py {}'.format(params))
+ sudo('python3 /tmp/configure_billing.py {}'.format(params))
try:
if os.environ['conf_stepcerts_enabled'] == 'true':
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
index d20f06b..3eabe20 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
index 22434f0..5542c44 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_instance.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_instance.py
index d5971fd..a626c03 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_instance.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_notebook_image.py
index 65da8f6..a1ac88b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_notebook_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
index b804e8a..0fe2fdc 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
index c3eae52..2794e9c 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_security_group.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_security_group.py
index f06bf41..b15a1b9 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_security_group.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_security_group.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
index 6bc4598..47df768 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_download_git_certfile.py b/infrastructure-provisioning/src/general/scripts/aws/common_download_git_certfile.py
index 2e5efeb..70df84a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_download_git_certfile.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_download_git_certfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
index ec2e42d..218b80c 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine.py b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine.py
index c1f53b9..be704cd 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index a672eff..bd61362 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_put_to_bucket.py b/infrastructure-provisioning/src/general/scripts/aws/common_put_to_bucket.py
index 9ccb627..66ea63b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_put_to_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_put_to_bucket.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_remove_remote_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/common_remove_remote_kernels.py
index 00b7900..d8c438d 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_remove_remote_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_remove_remote_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_reupload_key.py b/infrastructure-provisioning/src/general/scripts/aws/common_reupload_key.py
index 24a4d6d..a31d8f3 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_reupload_key.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_reupload_key.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
index 571e81e..e60bca1 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_stop_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_stop_notebook.py
index f890f40..0962bb1 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_stop_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_stop_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
index ca0af77..5a90d1b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook_image.py b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook_image.py
index 29a4fa9..e076c51 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
index eed5b4a..ef5d0ac 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
index 86c430c..a2fb168 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -119,9 +119,9 @@
"--sse AES256 --endpoint-url {6} --region {2}, " \
"ActionOnFailure=TERMINATE_CLUSTER,Jar=command-runner.jar;" \
"Name=CUSTOM_JAR, Args=sudo " \
- "/usr/bin/python /tmp/key_importer.py --user_name {4}, " \
+ "/usr/bin/python3 /tmp/key_importer.py --user_name {4}, " \
"ActionOnFailure=TERMINATE_CLUSTER,Jar=command-runner.jar; " \
- "Name=CUSTOM_JAR, Args=/usr/bin/python /tmp/jars_parser.py " \
+ "Name=CUSTOM_JAR, Args=/usr/bin/python3 /tmp/jars_parser.py " \
"--bucket {0} --emr_version {3} --region {2} --user_name {4} " \
"--cluster_name {5}, " \
"ActionOnFailure=TERMINATE_CLUSTER,Jar=command-runner.jar".\
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
index 04a9107..2354a66 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
index 699f07c..660dc48 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_key_importer.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_key_importer.py
index 8d5efd4..a4d4a1c 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_key_importer.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_key_importer.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
index 0a80431..9aca247 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
index 94785ac..0ef0c88 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_terminate.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_terminate.py
index 0820e39..db9077c 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
index d34c94e..8abf5d4 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
index 6f930fb..ca68a33 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
index 60b0e3e..57f74fa 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine_stop.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine_stop.py
index 1d540d9..8f849ab 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine_terminate.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine_terminate.py
index 84e9b3d..f0bfe5a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
index 40974c7..e5cefbc 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_associate_elastic_ip.py b/infrastructure-provisioning/src/general/scripts/aws/edge_associate_elastic_ip.py
index 5363073..8f5eb71 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_associate_elastic_ip.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_associate_elastic_ip.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
index 91c1dde..251687a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
index e034ce0..da5eb5e 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_start.py b/infrastructure-provisioning/src/general/scripts/aws/edge_start.py
index 557b534..893fb89 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_status.py b/infrastructure-provisioning/src/general/scripts/aws/edge_status.py
index 5982ad1..9c714b5 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_status.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_status.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_stop.py b/infrastructure-provisioning/src/general/scripts/aws/edge_stop.py
index 63c1e86..062e8d3 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index 17a33a2..05413d4 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
index 4070c3a..1e4bfe3 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
index 0cfbd5c..fa2d60d 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -62,13 +62,14 @@
sudo(
'\cp /tmp/jupyter_dataengine-service_create_configs.py /usr/local/bin/jupyter_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/jupyter_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
@@ -86,15 +87,15 @@
'/tmp/scala_version')
s3_client.download_file(args.bucket, args.project_name + '/' + args.cluster_name + '/python_version',
'/tmp/python_version')
- with file('/tmp/scala_version') as f:
+ with open('/tmp/scala_version') as f:
scala_version = str(f.read()).rstrip()
print(scala_version)
- with file('/tmp/python_version') as f:
+ with open('/tmp/python_version') as f:
python_version = str(f.read()).rstrip()
print(python_version)
if r_enabled == 'true':
s3_client.download_file(args.bucket, args.project_name + '/' + args.cluster_name + '/r_version', '/tmp/r_version')
- with file('/tmp/r_version') as g:
+ with open('/tmp/r_version') as g:
r_version = str(g.read()).rstrip()
print(r_version)
else:
@@ -102,7 +103,7 @@
cluster_id = get_emr_id_by_name(args.cluster_name)
master_instances = get_emr_instances_list(cluster_id, 'MASTER')
master_ip = master_instances[0].get('PrivateIpAddress')
- sudo("/usr/bin/python /usr/local/bin/jupyter_dataengine-service_create_configs.py --bucket " + args.bucket
+ sudo("/usr/bin/python3 /usr/local/bin/jupyter_dataengine-service_create_configs.py --bucket " + args.bucket
+ " --cluster_name " + args.cluster_name + " --emr_version " + args.emr_version + " --spark_version "
+ spark_version + " --scala_version " + scala_version + " --r_version " + r_version + " --hadoop_version "
+ hadoop_version + " --region " + args.region + " --excluded_lines '" + args.emr_excluded_spark_properties
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
index bdace86..7d0ac43 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py b/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
index ed3ce25..49d6393 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
index 15dbfb9..b0101fd 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 14012ed..f65b64e 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_terminate.py b/infrastructure-provisioning/src/general/scripts/aws/project_terminate.py
index 6db396e..88e7aa8 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
index 1e6cdb1..4fe2516 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_dataengine-service_create_configs.py
index d4f38e7..01850a2 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_dataengine-service_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
index ee7800b..111978a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -51,13 +51,14 @@
sudo(
'\cp /tmp/rstudio_dataengine-service_create_configs.py /usr/local/bin/rstudio_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/rstudio_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
@@ -68,7 +69,7 @@
configure_notebook(args)
spark_version = get_spark_version(args.cluster_name)
hadoop_version = get_hadoop_version(args.cluster_name)
- sudo("/usr/bin/python /usr/local/bin/rstudio_dataengine-service_create_configs.py --bucket " + args.bucket +
+ sudo("/usr/bin/python3 /usr/local/bin/rstudio_dataengine-service_create_configs.py --bucket " + args.bucket +
" --cluster_name " + args.cluster_name + " --emr_version " + args.emr_version + " --spark_version " +
spark_version + " --hadoop_version " + hadoop_version + " --region " + args.region + " --excluded_lines '"
+ args.emr_excluded_spark_properties + "' --project_name " + args.project_name + " --os_user " + args.os_user)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_associate_elastic_ip.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_associate_elastic_ip.py
index bc08f5f..6615744 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_associate_elastic_ip.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_associate_elastic_ip.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index 64e41a4..29b86c9 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_endpoint.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_endpoint.py
index f54b89f..19a17d5 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_endpoint.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_endpoint.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
index 6042606..afaea63 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_finalize.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_finalize.py
index e85379e..31ae0a3a100ae 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_finalize.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_finalize.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 865cb9c..8c545b3 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate.py
index 13c8434..86c2ee0 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate_aws_resources.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate_aws_resources.py
index f2f992f..72ad38f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate_aws_resources.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate_aws_resources.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
index b813eec..b201ba1 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
index 044efdc..83aa12b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
index dbee6a8..b2e413c 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
index ca1e9d8..a8d5fa5 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
index f6b09b9..9469ac1 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -57,13 +57,14 @@
sudo(
'\cp /tmp/zeppelin_dataengine-service_create_configs.py /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
@@ -77,7 +78,7 @@
livy_version = os.environ['notebook_livy_version']
r_enabled = os.environ['notebook_r_enabled']
numpy_version = os.environ['notebook_numpy_version']
- command = "/usr/bin/python /usr/local/bin/zeppelin_dataengine-service_create_configs.py " \
+ command = "/usr/bin/python3 /usr/local/bin/zeppelin_dataengine-service_create_configs.py " \
"--bucket {0} " \
"--cluster_name {1} " \
"--emr_version {2} " \
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
index 10d3dfa..24984f5 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_datalake_directory.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_datalake_directory.py
index 55c110b..29ff0c2 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_datalake_directory.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_datalake_directory.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_instance.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_instance.py
index 9ae95a6..5ad8253 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_instance.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
index fa31423..5866a65 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
index f39a5de..05b6a8b 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
index 922b993..04f47bd 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_create_subnet.py b/infrastructure-provisioning/src/general/scripts/azure/common_create_subnet.py
index 3bc237f..0698645 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_subnet.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_subnet.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py b/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
index a291925..718fb71 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
index c0cb7fe..9167703 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py
index 4add786..7795847 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
index 2d69a1f..a23b6a2 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_remove_remote_kernels.py b/infrastructure-provisioning/src/general/scripts/azure/common_remove_remote_kernels.py
index 51f900c..a6e4696 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_remove_remote_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_remove_remote_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py b/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
index fa4a5d5..795d753 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
index dd59414..3f153fb 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -93,7 +93,7 @@
put('~/scripts/common_notebook_update_refresh_token.py', '/tmp/common_notebook_update_refresh_token.py')
sudo('mv /tmp/common_notebook_update_refresh_token.py '
'/usr/local/bin/common_notebook_update_refresh_token.py')
- sudo("/usr/bin/python /usr/local/bin/{}.py {}".format('common_notebook_update_refresh_token', params))
+ sudo("/usr/bin/python3 /usr/local/bin/{}.py {}".format('common_notebook_update_refresh_token', params))
except:
traceback.print_exc()
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py b/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
index 8344590..82a9533 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
index 072132b..d2b8216 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
index 6bcd753..295ecab 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index c30e00e..f08cd05 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
index 41894ec..fcfd763 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
index 04b2009..c216bcb 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
index d03dd52..9db2c9b 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
index b78b63f..7cec539 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
index 751b679..50f5d0f 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
index fdf4b3a..9b26e74 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
index fafcbb1..7278bfb 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_start.py b/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
index 1026bbc..2f9f2ba 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_status.py b/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
index dc4a11e..4e87ce8 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py b/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
index 7b5a53f..d2bbf5c 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
index a10b1e8..f7c470a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index ca24f35..2d62cd1 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
index dde1bc6..b2a3631 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index 7e6650e..82f3392 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
index 86e7917..cfe04db 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_change_pass.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_change_pass.py
index 6af1829..2e57e90 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_change_pass.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_change_pass.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
index 0fc9082..d603098 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
index 5142136..17a5655 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -149,7 +149,7 @@
logging.info('[INSTALLING PREREQUISITES TO SSN INSTANCE]')
print('[INSTALLING PREREQUISITES TO SSN INSTANCE]')
params = "--hostname {} --keyfile {} --pip_packages 'backoff bcrypt==3.1.7 argparse fabric==1.14.0 pymongo pyyaml " \
- "pycrypto azure==2.0.0' --user {} --region {}".format(ssn_conf['instance_host'],
+ "pycryptodome azure==2.0.0' --user {} --region {}".format(ssn_conf['instance_host'],
ssn_conf['ssh_key_path'],
ssn_conf['datalab_ssh_user'],
ssn_conf['region'])
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_datalake.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_datalake.py
index 6f9ffc7..471d78c 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_datalake.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_datalake.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_peering.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_peering.py
index 4f2d342..a827c0a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_peering.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_peering.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_resource_group.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_resource_group.py
index c6f761c..4463350 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_resource_group.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_resource_group.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_vpc.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_vpc.py
index faeefbd..015bace 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_vpc.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_vpc.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
index cf6433a..38f27ca 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
index 7c7399a..753d938 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
index 5507644..7055a5e 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
index 7f1a90b..65448d0 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
index 1464a0d..9c75f91 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_collect_data.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
index 3dfeda9..061746a 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_firewall.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_firewall.py
index 7874352..aa126c5 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_firewall.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_firewall.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
index 3b85760..decdaf9 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_nat_route.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_nat_route.py
index 435eafb..d9a5f0b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_nat_route.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_nat_route.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py
index 97dac1f..f7ba4fd 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_service_account.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_service_account.py
index 8d46793..d1e9a55 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_service_account.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_service_account.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_subnet.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_subnet.py
index b35d277..e66866d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_subnet.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_subnet.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -59,12 +59,12 @@
sorted_subnets_cidr = sorted(subnets_cidr, key=sortkey)
if not empty_vpc:
- last_ip = int(ipaddress.IPv4Address(sorted_subnets_cidr[0].split('/')[0].decode("utf-8")))
+ last_ip = int(ipaddress.IPv4Address(sorted_subnets_cidr[0].split('/')[0]))
else:
- last_ip = int(ipaddress.IPv4Address(args.vpc_cidr.split('/')[0].decode("utf-8")))
+ last_ip = int(ipaddress.IPv4Address(args.vpc_cidr.split('/')[0]))
previous_subnet_size = private_subnet_size
for cidr in sorted_subnets_cidr:
- first_ip = int(ipaddress.IPv4Address(cidr.split('/')[0].decode("utf-8")))
+ first_ip = int(ipaddress.IPv4Address(cidr.split('/')[0]))
if first_ip - last_ip < private_subnet_size or previous_subnet_size < private_subnet_size:
subnet_size = ipaddress.ip_network(u'{}'.format(cidr)).num_addresses
last_ip = first_ip + subnet_size - 1
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_download_git_certfile.py b/infrastructure-provisioning/src/general/scripts/gcp/common_download_git_certfile.py
index b01b33e..256a65f 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_download_git_certfile.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_download_git_certfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
index fb45cf2..b213bdf 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -74,7 +74,7 @@
else:
application = os.environ['application']
- additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "" ).replace(
+ additional_tags = os.environ['tags'].replace("': '", ":").replace("', '", ",").replace("{'", "" ).replace(
"'}", "").lower()
notebook_config['cluster_labels'] = {
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
index a794f5b..f721f8c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
index b2f1b44..a2818fe 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -154,7 +154,8 @@
data = {"notebook_name": notebook_config['instance_name'], "error": ""}
json.dump(data, f)
- additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "" ).replace(
+ print('Additional tags will be added: {}'.format(os.environ['tags']))
+ additional_tags = os.environ['tags'].replace("': '", ":").replace("', '", ",").replace("{'", "" ).replace(
"'}", "").lower()
print('Additional tags will be added: {}'.format(additional_tags))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_put_to_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_put_to_bucket.py
index 17b5413..85a0d0e 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_put_to_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_put_to_bucket.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py b/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py
index 8c50902..25089af 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_reupload_key.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
index 8a99990..c41a1a8 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_start_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py
index 29d9c79..094de05 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_stop_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
index 014cc47..100d49d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook_image.py b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook_image.py
index ec7d52b..a448841 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook_image.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
index cf68612..7008175 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -61,7 +61,7 @@
env['connection_attempts'] = 100
env.key_filename = "{}".format(dataproc_conf['key_path'])
env.host_string = dataproc_conf['datalab_ssh_user'] + '@' + dataproc_conf['instance_ip']
- datalab.notebook_lib.install_os_pkg([['python-pip', 'N/A'], ['python3-pip', 'N/A']])
+ datalab.notebook_lib.install_os_pkg([['python3-pip', 'N/A']])
datalab.fab.configure_data_engine_service_pip(dataproc_conf['instance_ip'],
dataproc_conf['datalab_ssh_user'],
dataproc_conf['key_path'])
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
index 42dd4ae..e169980 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
index 53f9992..b1d3c09 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_install_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py
index 04870f2..3c949a2 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_jars_parser.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_key_importer.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_key_importer.py
index c996937..2260e5c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_key_importer.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_key_importer.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
index 02cb0d6..03d2e36 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_list_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
index 3d604ac..0013f69 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -79,7 +79,7 @@
dataproc_conf['project_name'],
dataproc_conf['endpoint_name'])
dataproc_conf['release_label'] = os.environ['dataproc_version']
- additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "").replace(
+ additional_tags = os.environ['tags'].replace("': '", ":").replace("', '", ",").replace("{'", "").replace(
"'}", "").lower()
dataproc_conf['cluster_labels'] = {
@@ -139,7 +139,8 @@
local("echo Waiting for changes to propagate; sleep 10")
- dataproc_cluster = json.loads(open('/root/templates/dataengine-service_cluster.json').read().decode('utf-8-sig'))
+ dataproc_cluster = json.loads(open('/root/templates/dataengine-service_cluster.json').read())
+ print(dataproc_cluster)
dataproc_cluster['projectId'] = os.environ['gcp_project_id']
dataproc_cluster['clusterName'] = dataproc_conf['cluster_name']
dataproc_cluster['labels'] = dataproc_conf['cluster_labels']
@@ -161,7 +162,7 @@
dataproc_cluster['config']['softwareConfig']['imageVersion'] = dataproc_conf['release_label']
ssh_user_pubkey = open('{}{}.pub'.format(os.environ['conf_key_dir'], dataproc_conf['project_name'])).read()
key = RSA.importKey(open(dataproc_conf['key_path'], 'rb').read())
- ssh_admin_pubkey = key.publickey().exportKey("OpenSSH")
+ ssh_admin_pubkey = key.publickey().exportKey("OpenSSH").decode('UTF-8')
dataproc_cluster['config']['gceClusterConfig']['metadata']['ssh-keys'] = '{0}:{1}\n{0}:{2}'.format(
dataproc_conf['datalab_ssh_user'], ssh_user_pubkey, ssh_admin_pubkey)
dataproc_cluster['config']['gceClusterConfig']['tags'][0] = dataproc_conf['cluster_tag']
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_terminate.py
index 386256d..f66a67d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
index 260af61..d282097 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index 1fd2f28..ef6ac25 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -152,7 +152,7 @@
data_engine['gpu_accelerator_type'] = os.environ['gcp_gpu_accelerator_type']
data_engine['network_tag'] = '{0}-{1}-{2}-ps'.format(data_engine['service_base_name'],
data_engine['project_name'], data_engine['endpoint_name'])
- additional_tags = os.environ['tags'].replace("': u'", ":").replace("', u'", ",").replace("{u'", "").replace(
+ additional_tags = os.environ['tags'].replace("': '", ":").replace("', '", ",").replace("{'", "").replace(
"'}", "").lower()
data_engine['slave_labels'] = {"name": data_engine['cluster_name'],
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
index 49b4f0c..88513b5 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py
index 15ee522..1a06c2d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py
index eebbce7..5751014 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
index 66b8bc4..b1c2496 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py
index 3a1ca91..6532870 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_create_static_ip.py b/infrastructure-provisioning/src/general/scripts/gcp/edge_create_static_ip.py
index dfd1b40..0411f7e 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_create_static_ip.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_create_static_ip.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py b/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py
index 7ea23db..eddcfa2 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_status.py b/infrastructure-provisioning/src/general/scripts/gcp/edge_status.py
index bd5f94b..8bc558a 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_status.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_status.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py b/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py
index 8a69179..e8ac3f3 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_stop.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py
index 6ba7e17..d92b697 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/edge_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index f8c99cf..814259d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
index 26f1a53..62e0277 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_dataengine-service_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index 543668d..e6353e6 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -60,19 +60,20 @@
# put(templates_dir + 'run_template.sh', '/tmp/run_template.sh')
sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
sudo('chmod 755 /usr/local/bin/create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
GCPActions().get_from_bucket(args.bucket, '{0}/{1}/scala_version'.format(args.project_name, args.cluster_name),
'/tmp/scala_version')
- with file('/tmp/scala_version') as f:
+ with open('/tmp/scala_version') as f:
scala_version = str(f.read()).replace(',', '')
env.hosts = "{}".format(args.notebook_ip)
env.user = args.os_user
@@ -93,7 +94,7 @@
sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf')
sudo('echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc')
sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; '
- '/usr/bin/python /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4}'
+ '/usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4}'
' --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} '
'--application {11} --r_version {12} --r_enabled {13} --python_version {14} --master_ip {15} --scala_version {16}'
.format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name,
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
index afb3c75..34d634f 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 47a781d..683cb1f 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py
index 4b8581b..7e0dbfc 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
index 7f5355a..f1831d7 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py
index 56e9c8e..d633f81 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
index 7431488..d296646 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -51,13 +51,14 @@
put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application), '/tmp/create_configs.py')
sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
sudo('chmod 755 /usr/local/bin/create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
@@ -70,6 +71,6 @@
hadoop_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop')
sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf')
sudo('echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc')
- sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11}'
+ sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11}'
.format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name, args.dataproc_version, spark_version, hadoop_version,
args.region, args.project_name, args.os_user, args.pip_mirror, args.application))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index 72d24ca..bd7d414 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -45,6 +45,7 @@
def clear_resources():
GCPActions.remove_instance(ssn_conf['instance_name'], ssn_conf['zone'])
+ GCPActions.remove_static_address(ssn_conf['static_address_name'], ssn_conf['region'])
GCPActions.remove_service_account(ssn_conf['service_account_name'], ssn_conf['service_base_name'])
GCPActions.remove_role(ssn_conf['role_name'])
if not ssn_conf['pre_defined_firewall']:
@@ -72,6 +73,7 @@
ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] = datalab.fab.replace_multi_symbols(
os.environ['conf_service_base_name'].replace('_', '-').lower()[:20], '-', True)
ssn_conf['instance_name'] = '{}-ssn'.format(ssn_conf['service_base_name'])
+ ssn_conf['static_address_name'] = '{}-ssn-static-ip'.format(ssn_conf['service_base_name'])
ssn_conf['role_name'] = '{}-{}-ssn-role'.format(ssn_conf['service_base_name'], ssn_conf['ssn_unique_index'])
ssn_conf['region'] = os.environ['gcp_region']
ssn_conf['zone'] = os.environ['gcp_zone']
@@ -159,8 +161,8 @@
logging.info('[INSTALLING PREREQUISITES TO SSN INSTANCE]')
print('[INSTALLING PREREQUISITES TO SSN INSTANCE]')
params = "--hostname {} --keyfile {} --pip_packages " \
- "'boto3 bcrypt==3.1.7 backoff argparse fabric==1.14.0 awscli pymongo pyyaml " \
- "google-api-python-client google-cloud-storage pycrypto' --user {} --region {}". \
+ "'boto3 bcrypt==3.1.7 backoff argparse fab-classic awscli pymongo pyyaml " \
+ "google-api-python-client google-cloud-storage pycryptodome' --user {} --region {}". \
format(ssn_conf['instance_hostname'], ssn_conf['ssh_key_path'],
ssn_conf['datalab_ssh_user'], ssn_conf['region'])
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_static_ip.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_static_ip.py
index 9f97d45..58bf787 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_static_ip.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_static_ip.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_vpc.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_vpc.py
index 2a450fc..d6bd97d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_vpc.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_create_vpc.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_finalize.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_finalize.py
index d0155c5..93761f1 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_finalize.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_finalize.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
index 9efa053..e95372f 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -113,7 +113,7 @@
try:
ssn_conf['vpc_selflink'] = GCPMeta.get_vpc(ssn_conf['vpc_name'])['selfLink']
- if os.environ['gcp_subnet_name'] == '':
+ if 'gcp_subnet_name' not in os.environ:
raise KeyError
else:
ssn_conf['pre_defined_subnet'] = True
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
index 29cddfd..b174a14 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate_gcp_resources.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate_gcp_resources.py
index 5119690..5464ec3 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate_gcp_resources.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_terminate_gcp_resources.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
index 9b53366..322cd10 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
index 545fdd3..dd39cae 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
index acac8c8..41ee2fc 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
index c96ca81..f6150e2 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py
index 8e94672..2c05849 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
index 774e001..e25625e 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -56,13 +56,14 @@
put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application), '/tmp/create_configs.py')
sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
sudo('chmod 755 /usr/local/bin/create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py b/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py
index dc6ed83..8f079c9 100644
--- a/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py
+++ b/infrastructure-provisioning/src/general/scripts/os/check_inactivity.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py b/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py
index da8376d..6053071 100644
--- a/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/os/common_clean_instance.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -55,7 +55,6 @@
def clean_jupyter():
try:
sudo('systemctl stop jupyter-notebook')
- sudo('pip2 uninstall -y notebook jupyter')
sudo('pip3 uninstall -y notebook jupyter')
sudo('rm -rf /usr/local/share/jupyter/')
sudo('rm -rf /home/{}/.jupyter/'.format(args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/os/common_configure_proxy.py b/infrastructure-provisioning/src/general/scripts/os/common_configure_proxy.py
index 8887db2..da04351 100644
--- a/infrastructure-provisioning/src/general/scripts/os/common_configure_proxy.py
+++ b/infrastructure-provisioning/src/general/scripts/os/common_configure_proxy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py b/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
index a89ddde..e7c644e 100644
--- a/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
+++ b/infrastructure-provisioning/src/general/scripts/os/common_configure_reverse_proxy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py b/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
index dea8e16..b48aa57 100644
--- a/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
+++ b/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py b/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
index 5a2f785..68c9a18 100644
--- a/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
+++ b/infrastructure-provisioning/src/general/scripts/os/configure_proxy_for_docker.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py b/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py
index 85eb8b9..f1b6127 100644
--- a/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/dataengine_install_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py b/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py
index e4d96f5..886fcf9 100644
--- a/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/dataengine_list_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/dataengine_reconfigure_spark.py b/infrastructure-provisioning/src/general/scripts/os/dataengine_reconfigure_spark.py
index c580a95..5dc572a 100644
--- a/infrastructure-provisioning/src/general/scripts/os/dataengine_reconfigure_spark.py
+++ b/infrastructure-provisioning/src/general/scripts/os/dataengine_reconfigure_spark.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
index 95d40cf..68c3ae0 100644
--- a/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -62,10 +62,10 @@
text = text.replace('CLUSTER_NAME', args.cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '2.7')
- text = text.replace('PYTHON_FULL_VERSION', '2.7')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_PATH', '/usr/bin/python2.7')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -83,9 +83,9 @@
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -101,7 +101,7 @@
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
- pyspark_kernel_name = 'PySpark (Python-3.6 / Spark-{0} ) [{1}]'.format(args.spark_version,
+ pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
pyspark_kernel_name, args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
index c69a055..9279198 100644
--- a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -58,12 +58,13 @@
put(scripts_dir + 'deeplearning_dataengine_create_configs.py',
'/usr/local/bin/deeplearning_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/deeplearning_dataengine_create_configs.py')
- if not exists('/usr/lib/python2.7/datalab/'):
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
- put('/usr/lib/python2.7/datalab/*', '/usr/lib/python2.7/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python2.7/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
@@ -82,7 +83,7 @@
create_inactivity_log(args.spark_master_ip, env.host_string)
if 'spark_configurations' not in os.environ:
os.environ['spark_configurations'] = '[]'
- sudo('/usr/bin/python /usr/local/bin/deeplearning_dataengine_create_configs.py '
+ sudo('/usr/bin/python3 /usr/local/bin/deeplearning_dataengine_create_configs.py '
'--cluster_name {} --spark_version {} --hadoop_version {} --os_user {} --spark_master {} --region {} '
'--datalake_enabled {} --spark_configurations "{}"'.format(args.cluster_name, args.spark_version,
args.hadoop_version, args.os_user, args.spark_master,
diff --git a/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py b/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
index a19243b..67ca9c9 100644
--- a/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -24,7 +24,7 @@
import argparse
import json
import sys
-import xmlrpclib
+import xmlrpc.client
from datalab.fab import *
from datalab.notebook_lib import *
from fabric.api import *
@@ -41,7 +41,7 @@
try:
for _ in range(100):
pip_pkgs = dict()
- client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
+ client = xmlrpc.client.ServerProxy('https://pypi.python.org/pypi')
raw_pkgs = client.browse(["Programming Language :: Python :: " + version + ""])
all_pkgs = [i[0] for i in raw_pkgs]
if len(all_pkgs) != 0:
@@ -59,7 +59,7 @@
def get_uncategorised_pip_pkgs(all_pkgs_pip2, all_pkgs_pip3):
try:
pip_pkgs = dict()
- client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
+ client = xmlrpc.client.ServerProxy('https://pypi.python.org/pypi')
raw_pkgs = client.list_packages()
all_pkgs_other = []
for pkg in raw_pkgs:
@@ -85,10 +85,10 @@
elif args.group == 'pip2':
all_pkgs['pip2'] = get_available_pip_pkgs("2.7")
elif args.group == 'pip3':
- all_pkgs['pip3'] = get_available_pip_pkgs("3.6")
+ all_pkgs['pip3'] = get_available_pip_pkgs("3.8")
elif args.group == 'others':
all_pkgs['pip2'] = get_available_pip_pkgs("2.7")
- all_pkgs['pip3'] = get_available_pip_pkgs("3.6")
+ all_pkgs['pip3'] = get_available_pip_pkgs("3.8")
all_pkgs['others'] = get_uncategorised_pip_pkgs(all_pkgs['pip2'], all_pkgs['pip3'])
elif args.group == 'r_pkg':
all_pkgs['r_pkg'] = get_available_r_pkgs()
diff --git a/infrastructure-provisioning/src/general/scripts/os/git_pre_commit.py b/infrastructure-provisioning/src/general/scripts/os/git_pre_commit.py
index 9987820..fb6d7ad 100644
--- a/infrastructure-provisioning/src/general/scripts/os/git_pre_commit.py
+++ b/infrastructure-provisioning/src/general/scripts/os/git_pre_commit.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
index 00282c4..0cbbd11 100644
--- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -94,18 +94,10 @@
pass
try:
- print('Installing other packages: {}'.format(pkgs['libraries']['others']))
+ print('Installing other packages (only tries pip3): {}'.format(pkgs['libraries']['others']))
for pkg in pkgs['libraries']['others']:
- if os.environ['conf_resource'] in ('dataengine-service'):#, 'dataengine'):
- status_pip3 = install_pip_pkg([pkg], 'pip3', 'others')
- general_status = general_status + status_pip3
- else:
- status_pip2 = install_pip_pkg([pkg], 'pip2', 'others')
- status_pip3 = install_pip_pkg([pkg], 'pip3', 'others')
- if status_pip2[0]['status'] == 'installed':
- general_status = general_status + status_pip2
- else:
- general_status = general_status + status_pip3
+ status_pip3 = install_pip_pkg([pkg], 'pip3', 'others')
+ general_status = general_status + status_pip3
except KeyError:
pass
diff --git a/infrastructure-provisioning/src/general/scripts/os/ipynb_output_filter.py b/infrastructure-provisioning/src/general/scripts/os/ipynb_output_filter.py
index 20587d7..6ca06c0 100644
--- a/infrastructure-provisioning/src/general/scripts/os/ipynb_output_filter.py
+++ b/infrastructure-provisioning/src/general/scripts/os/ipynb_output_filter.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
index bf29578..1c9328d 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -117,10 +117,10 @@
text = text.replace('CLUSTER_NAME', args.cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '2.7')
- text = text.replace('PYTHON_FULL_VERSION', '2.7')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_PATH', '/usr/bin/python2.7')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -138,9 +138,9 @@
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -156,7 +156,7 @@
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir))
- pyspark_kernel_name = 'PySpark (Python-3.6 / Spark-{0} ) [{1}]'.format(args.spark_version,
+ pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
pyspark_kernel_name, args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
index d7505d5..d41cde7 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -62,12 +62,13 @@
put(scripts_dir + 'jupyter_dataengine_create_configs.py', '/usr/local/bin/jupyter_dataengine_create_configs.py',
use_sudo=True)
sudo('chmod 755 /usr/local/bin/jupyter_dataengine_create_configs.py')
- if not exists('/usr/lib/python2.7/datalab/'):
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
- put('/usr/lib/python2.7/datalab/*', '/usr/lib/python2.7/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python2.7/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
@@ -87,7 +88,7 @@
os.environ['spark_configurations'] = '[]'
configure_notebook(args.keyfile, env.host_string)
create_inactivity_log(args.spark_master_ip, env.host_string)
- sudo('/usr/bin/python /usr/local/bin/jupyter_dataengine_create_configs.py '
+ sudo('/usr/bin/python3 /usr/local/bin/jupyter_dataengine_create_configs.py '
'--cluster_name {} --spark_version {} --hadoop_version {} --os_user {} \
--spark_master {} --datalake_enabled {} --r_enabled {} --spark_configurations "{}"'.
format(args.cluster_name, args.spark_version, args.hadoop_version, args.os_user, args.spark_master,
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
index 06a71cf..17db0ba 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/manage_git_creds.py b/infrastructure-provisioning/src/general/scripts/os/manage_git_creds.py
index b2f8375..f0a73c0 100644
--- a/infrastructure-provisioning/src/general/scripts/os/manage_git_creds.py
+++ b/infrastructure-provisioning/src/general/scripts/os/manage_git_creds.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/notebook_git_creds.py b/infrastructure-provisioning/src/general/scripts/os/notebook_git_creds.py
index e66b881..75382f6 100644
--- a/infrastructure-provisioning/src/general/scripts/os/notebook_git_creds.py
+++ b/infrastructure-provisioning/src/general/scripts/os/notebook_git_creds.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py b/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py
index bb1c07e..6c26550 100644
--- a/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py
+++ b/infrastructure-provisioning/src/general/scripts/os/notebook_inactivity_check.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/notebook_install_libs.py b/infrastructure-provisioning/src/general/scripts/os/notebook_install_libs.py
index 879911c..17fb635 100644
--- a/infrastructure-provisioning/src/general/scripts/os/notebook_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/notebook_install_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py b/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py
index 849189d..60bcd62 100644
--- a/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/notebook_list_libs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_dataengine_spark.py b/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_dataengine_spark.py
index c8b46b9..1491e14 100644
--- a/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_dataengine_spark.py
+++ b/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_dataengine_spark.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_spark.py b/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_spark.py
index 86b3a7f..c08bdf8 100644
--- a/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_spark.py
+++ b/infrastructure-provisioning/src/general/scripts/os/notebook_reconfigure_spark.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py b/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py
index 18a95f2..2df08f0 100644
--- a/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py
+++ b/infrastructure-provisioning/src/general/scripts/os/reconfigure_spark.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -67,6 +67,6 @@
datalake_enabled = 'false'
if 'spark_configurations' not in os.environ:
os.environ['spark_configurations'] = '[]'
- sudo('/usr/bin/python /usr/local/bin/notebook_reconfigure_dataengine_spark.py --cluster_name {0} '
+ sudo('/usr/bin/python3 /usr/local/bin/notebook_reconfigure_dataengine_spark.py --cluster_name {0} '
'--jars_dir {1} --cluster_dir {2} --datalake_enabled {3} --spark_configurations "{4}"'.format(
args.cluster_name, jars_dir, cluster_dir, datalake_enabled, os.environ['spark_configurations']))
diff --git a/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py
index beffebe..17022a4 100644
--- a/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/rstudio_dataengine_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
index 4446c69..18c6313 100644
--- a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -55,12 +55,13 @@
put(scripts_dir + 'rstudio_dataengine_create_configs.py', '/usr/local/bin/rstudio_dataengine_create_configs.py',
use_sudo=True)
sudo('chmod 755 /usr/local/bin/rstudio_dataengine_create_configs.py')
- if not exists('/usr/lib/python2.7/datalab/'):
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
- put('/usr/lib/python2.7/datalab/*', '/usr/lib/python2.7/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python2.7/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
@@ -79,7 +80,7 @@
os.environ['spark_configurations'] = '[]'
configure_notebook(args.keyfile, env.host_string)
create_inactivity_log(args.spark_master_ip, env.host_string)
- sudo('/usr/bin/python /usr/local/bin/rstudio_dataengine_create_configs.py '
+ sudo('/usr/bin/python3 /usr/local/bin/rstudio_dataengine_create_configs.py '
'--cluster_name {} --spark_version {} --hadoop_version {} --os_user {} --spark_master {} --region {} '
'--datalake_enabled {} --spark_configurations "{}"'.
format(args.cluster_name, args.spark_version, args.hadoop_version, args.os_user, args.spark_master, region,
diff --git a/infrastructure-provisioning/src/general/scripts/os/superset_start.py b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
index 90d1696..5207dbb 100644
--- a/infrastructure-provisioning/src/general/scripts/os/superset_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py
index 5504fa0..24bc6d7 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_dataengine_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
index 7e0df9c..3766073 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -55,12 +55,13 @@
put(scripts_dir + 'tensor-rstudio_dataengine_create_configs.py',
'/usr/local/bin/tensor-rstudio_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/tensor-rstudio_dataengine_create_configs.py')
- if not exists('/usr/lib/python2.7/datalab/'):
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
- put('/usr/lib/python2.7/datalab/*', '/usr/lib/python2.7/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python2.7/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
@@ -79,7 +80,7 @@
os.environ['spark_configurations'] = '[]'
configure_notebook(args.keyfile, env.host_string)
create_inactivity_log(args.spark_master_ip, env.host_string)
- sudo('/usr/bin/python /usr/local/bin/tensor-rstudio_dataengine_create_configs.py '
+ sudo('/usr/bin/python3 /usr/local/bin/tensor-rstudio_dataengine_create_configs.py '
'--cluster_name {} --spark_version {} --hadoop_version {} --os_user {} --spark_master {} --region {} '
'--datalake_enabled {} --spark_configurations "{}"'.
format(args.cluster_name, args.spark_version, args.hadoop_version, args.os_user, args.spark_master, region,
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
index 61437c8..e93d125 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -62,10 +62,10 @@
text = text.replace('CLUSTER_NAME', args.cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '2.7')
- text = text.replace('PYTHON_FULL_VERSION', '2.7')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_PATH', '/usr/bin/python2.7')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -83,9 +83,9 @@
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -101,7 +101,7 @@
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
- pyspark_kernel_name = 'PySpark (Python-3.6 / Spark-{0} ) [{1}]'.format(args.spark_version,
+ pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
pyspark_kernel_name, args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
index b3d70a4..c93a220 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -55,12 +55,13 @@
put(scripts_dir + 'tensor_dataengine_create_configs.py', '/usr/local/bin/tensor_dataengine_create_configs.py',
use_sudo=True)
sudo('chmod 755 /usr/local/bin/tensor_dataengine_create_configs.py')
- if not exists('/usr/lib/python2.7/datalab/'):
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
- put('/usr/lib/python2.7/datalab/*', '/usr/lib/python2.7/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python2.7/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
@@ -79,7 +80,7 @@
os.environ['spark_configurations'] = '[]'
configure_notebook(args.keyfile, env.host_string)
create_inactivity_log(args.spark_master_ip, env.host_string)
- sudo('/usr/bin/python /usr/local/bin/tensor_dataengine_create_configs.py '
+ sudo('/usr/bin/python3 /usr/local/bin/tensor_dataengine_create_configs.py '
'--cluster_name {} --spark_version {} --hadoop_version {} --os_user {} --spark_master {} --region {} '
'--datalake_enabled {} --spark_configurations "{}"'.
format(args.cluster_name, args.spark_version, args.hadoop_version, args.os_user, args.spark_master, region,
diff --git a/infrastructure-provisioning/src/general/scripts/os/update_inactivity_on_start.py b/infrastructure-provisioning/src/general/scripts/os/update_inactivity_on_start.py
index 723679e..445c71e 100644
--- a/infrastructure-provisioning/src/general/scripts/os/update_inactivity_on_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/update_inactivity_on_start.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
index 939e394..5c917a2 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -91,7 +91,7 @@
if os.path.exists(livy_path + 'conf/spark-blacklist.conf'):
local('sudo sed -i "s/^/#/g" ' + livy_path + 'conf/spark-blacklist.conf')
local(''' sudo echo "export SPARK_HOME=''' + cluster_dir + '''spark/" >> ''' + livy_path + '''conf/livy-env.sh''')
- local(''' sudo echo "export PYSPARK3_PYTHON=python3.6" >> ''' +
+ local(''' sudo echo "export PYSPARK3_PYTHON=python3.8" >> ''' +
livy_path + '''conf/livy-env.sh''')
template_file = "/tmp/{}/dataengine_interpreter.json".format(args.cluster_name)
fr = open(template_file, 'r+')
@@ -122,7 +122,7 @@
local('sudo systemctl start livy-server-' + str(livy_port))
else:
template_file = "/tmp/{}/dataengine_interpreter.json".format(args.cluster_name)
- p_versions = ["2", "3.6"]
+ p_versions = ["2", "3.8"]
for p_version in p_versions:
fr = open(template_file, 'r+')
text = fr.read()
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
index dc894b3..6d0b0f9 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -61,12 +61,13 @@
put(scripts_dir + 'zeppelin_dataengine_create_configs.py',
'/usr/local/bin/zeppelin_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/zeppelin_dataengine_create_configs.py')
- if not exists('/usr/lib/python2.7/datalab/'):
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
- put('/usr/lib/python2.7/datalab/*', '/usr/lib/python2.7/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python2.7/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
@@ -87,7 +88,7 @@
create_inactivity_log(args.spark_master_ip, env.host_string)
livy_version = os.environ['notebook_livy_version']
r_enabled = os.environ['notebook_r_enabled']
- sudo('/usr/bin/python /usr/local/bin/zeppelin_dataengine_create_configs.py '
+ sudo('/usr/bin/python3 /usr/local/bin/zeppelin_dataengine_create_configs.py '
'--cluster_name {} --spark_version {} --hadoop_version {} --os_user {} --spark_master {} --keyfile {} \
--notebook_ip {} --livy_version {} --multiple_clusters {} --region {} --datalake_enabled {} '
'--r_enabled {} --spark_configurations "{}"'.
diff --git a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
index e03c629..45c1213 100644
--- a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
+++ b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
@@ -7,9 +7,9 @@
"{connection_file}"
],
"language": "python",
- "display_name": "Local PySpark (Python-3.6 / Spark-SP_VER )",
+ "display_name": "Local PySpark (Python-3.8 / Spark-SP_VER )",
"env": {
- "PYSPARK_PYTHON": "python3.6",
+ "PYSPARK_PYTHON": "python3.8",
"SPARK_HOME": "/opt/spark/",
"PYTHONPATH": "PY4J:/opt/spark/python/:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
diff --git a/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json b/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json
index 56b0090..1001f0f 100644
--- a/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json
+++ b/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json
@@ -1,15 +1,15 @@
{
"argv": [
- "/usr/bin/python2.7",
+ "/usr/bin/python3.8",
"-m",
"ipykernel",
"-f",
"{connection_file}"
],
"language": "python",
- "display_name": "Local PySpark (Python-2.7 / Spark-SP_VER )",
+ "display_name": "Local PySpark (Python-3.8 / Spark-SP_VER )",
"env": {
- "PYSPARK_PYTHON": "python2.7",
+ "PYSPARK_PYTHON": "python3.8",
"SPARK_HOME": "/opt/spark/",
"PYTHONPATH": "PY4J:/opt/spark/python/:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
diff --git a/infrastructure-provisioning/src/jupyter/fabfile.py b/infrastructure-provisioning/src/jupyter/fabfile.py
index e5481b6..770ddc7 100644
--- a/infrastructure-provisioning/src/jupyter/fabfile.py
+++ b/infrastructure-provisioning/src/jupyter/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
index 74b33dd..c7fcfe7 100644
--- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
+++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -94,8 +94,6 @@
if os.environ['notebook_r_enabled'] == 'true':
print("Installing R")
ensure_r(args.os_user, r_libs, args.region, args.r_mirror)
- print("Install Python 2 modules")
- ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
diff --git a/infrastructure-provisioning/src/jupyterlab/fabfile.py b/infrastructure-provisioning/src/jupyterlab/fabfile.py
index 42822fc..4ce375f 100644
--- a/infrastructure-provisioning/src/jupyterlab/fabfile.py
+++ b/infrastructure-provisioning/src/jupyterlab/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py b/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py
index 0e317b2..0634fe2 100644
--- a/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py
+++ b/infrastructure-provisioning/src/jupyterlab/scripts/configure_jupyterlab_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/project/fabfile.py b/infrastructure-provisioning/src/project/fabfile.py
index 851e837..fc01dc6 100644
--- a/infrastructure-provisioning/src/project/fabfile.py
+++ b/infrastructure-provisioning/src/project/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
index 0adb505..3d49a2d 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_http_proxy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/project/scripts/configure_keycloak.py b/infrastructure-provisioning/src/project/scripts/configure_keycloak.py
index 1120759..11850c7 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_keycloak.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_keycloak.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/project/scripts/configure_nftables.py b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
index c630cab..872d3bd 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_nftables.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_nftables.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py b/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py
index edb86fd..322c642 100644
--- a/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py
+++ b/infrastructure-provisioning/src/project/scripts/configure_nginx_reverse_proxy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py b/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py
index bef9ec9..20dcd4c 100644
--- a/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py
+++ b/infrastructure-provisioning/src/project/scripts/reupload_ssh_key.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/rstudio/fabfile.py b/infrastructure-provisioning/src/rstudio/fabfile.py
index 2d9ff25..fa7149e 100644
--- a/infrastructure-provisioning/src/rstudio/fabfile.py
+++ b/infrastructure-provisioning/src/rstudio/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
index 9cb01a0..a383ff3 100644
--- a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
+++ b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -85,8 +85,6 @@
ensure_jre_jdk(args.os_user)
print("Install R")
ensure_r(args.os_user, r_libs, args.region, args.r_mirror)
- print("Install Python 2 modules")
- ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
diff --git a/infrastructure-provisioning/src/ssn/fabfile.py b/infrastructure-provisioning/src/ssn/fabfile.py
index a004520..47da688 100644
--- a/infrastructure-provisioning/src/ssn/fabfile.py
+++ b/infrastructure-provisioning/src/ssn/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/ssn/scripts/backup.py b/infrastructure-provisioning/src/ssn/scripts/backup.py
index cbf9a2b..67ea945 100644
--- a/infrastructure-provisioning/src/ssn/scripts/backup.py
+++ b/infrastructure-provisioning/src/ssn/scripts/backup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
index 0ef68c2..b5511d2 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py b/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py
index 33faca3..72c9ea2 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_conf_file.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
@@ -21,7 +21,7 @@
# ******************************************************************************
-import ConfigParser
+import configparser
import argparse
import json
import sys
@@ -49,7 +49,7 @@
with open('/tmp/datalab.ini.modified', 'w') as conf_file_modified:
conf_file_modified.writelines(conf_list)
- config = ConfigParser.RawConfigParser()
+ config = configparser.RawConfigParser()
config.read('/tmp/datalab.ini.modified')
for section in config.sections():
options = config.options(section)
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 3890612..d8d9794 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -54,7 +54,7 @@
local('scp -r -i {} /project_tree/* {}:{}sources/'.format(args.keyfile, env.host_string, args.datalab_path))
local('scp -i {} /root/scripts/configure_conf_file.py {}:/tmp/configure_conf_file.py'.format(args.keyfile,
env.host_string))
- sudo("python /tmp/configure_conf_file.py --datalab_dir {} --variables_list '{}'".format(
+ sudo("python3 /tmp/configure_conf_file.py --datalab_dir {} --variables_list '{}'".format(
args.datalab_path, json.dumps(variables_list)))
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py b/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py
index 0c12f21..1423734 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_gitlab.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py
index 3fc7334..f78dfc3 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
index 7aa9ab8..f60632b 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index fce5306..081b1e3 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -90,13 +90,14 @@
def copy_ssn_libraries():
try:
- sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
+ sudo('mkdir -p /usr/lib64/python3.8')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
except Exception as err:
traceback.print_exc()
print('Failed to copy ssn libraries: ', str(err))
@@ -133,8 +134,8 @@
sudo('sed -i "s|CLOUD_PROVIDER|{0}|g" /tmp/local_endpoint.json'.format(
os.environ['conf_cloud_provider'].upper()))
sudo('mv /tmp/local_endpoint.json ' + args.datalab_path + 'tmp/')
- sudo('pip2 install -U six>=1.13.0')
- sudo("python " + args.datalab_path + "tmp/configure_mongo.py --datalab_path {} ".format(
+ sudo('pip3 install -U six==1.15.0')
+ sudo("python3 " + args.datalab_path + "tmp/configure_mongo.py --datalab_path {} ".format(
args.datalab_path))
except Exception as err:
traceback.print_exc()
diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
index 5136b42..fc26bf8 100644
--- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py
+++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py b/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py
index d6b8dfc..047eac4 100644
--- a/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py
+++ b/infrastructure-provisioning/src/ssn/scripts/gitlab_deploy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
diff --git a/infrastructure-provisioning/src/ssn/scripts/resource_status.py b/infrastructure-provisioning/src/ssn/scripts/resource_status.py
index 3cd053b..7146076 100644
--- a/infrastructure-provisioning/src/ssn/scripts/resource_status.py
+++ b/infrastructure-provisioning/src/ssn/scripts/resource_status.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
diff --git a/infrastructure-provisioning/src/ssn/scripts/restore.py b/infrastructure-provisioning/src/ssn/scripts/restore.py
index e770f98..eeccf82 100644
--- a/infrastructure-provisioning/src/ssn/scripts/restore.py
+++ b/infrastructure-provisioning/src/ssn/scripts/restore.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -90,7 +90,7 @@
if ask("Maybe you want to create backup of existing configuration before restoring?"):
with settings(hide('everything')):
print("Creating new backup...")
- local("python backup.py --configs all --keys all --certs all --jar all --db")
+ local("python3 backup.py --configs all --keys all --certs all --jar all --db")
except:
print("Failed to create new backup.")
sys.exit(1)
diff --git a/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py b/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py
index 14c696a..4ad6cdd 100644
--- a/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py
+++ b/infrastructure-provisioning/src/ssn/scripts/upload_response_file.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/superset/fabfile.py b/infrastructure-provisioning/src/superset/fabfile.py
index 5f5fe63..952b0e6 100644
--- a/infrastructure-provisioning/src/superset/fabfile.py
+++ b/infrastructure-provisioning/src/superset/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
index 6dd0e3d..a764ff6 100644
--- a/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
+++ b/infrastructure-provisioning/src/superset/scripts/configure_superset_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/tensor-rstudio/fabfile.py b/infrastructure-provisioning/src/tensor-rstudio/fabfile.py
index 60906b5..a19f830 100644
--- a/infrastructure-provisioning/src/tensor-rstudio/fabfile.py
+++ b/infrastructure-provisioning/src/tensor-rstudio/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
index 7f65897..6b3a16a 100644
--- a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
+++ b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -93,8 +93,6 @@
ensure_jre_jdk(args.os_user)
print("Install R")
ensure_r(args.os_user, r_libs, args.region, args.r_mirror)
- print("Install Python 2 modules")
- ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
diff --git a/infrastructure-provisioning/src/tensor/fabfile.py b/infrastructure-provisioning/src/tensor/fabfile.py
index 6aaec8e..923f3c2 100644
--- a/infrastructure-provisioning/src/tensor/fabfile.py
+++ b/infrastructure-provisioning/src/tensor/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
index 2a1b3ac..83701e3 100644
--- a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
+++ b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -90,8 +90,6 @@
# INSTALL LANGUAGES
print("Install Java")
ensure_jre_jdk(args.os_user)
- print("Install Python 2 modules")
- ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
diff --git a/infrastructure-provisioning/src/zeppelin/fabfile.py b/infrastructure-provisioning/src/zeppelin/fabfile.py
index f7753d4..63965f7 100644
--- a/infrastructure-provisioning/src/zeppelin/fabfile.py
+++ b/infrastructure-provisioning/src/zeppelin/fabfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
index 3d8c254..843d7f8 100644
--- a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
+++ b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
@@ -221,8 +221,6 @@
if os.environ['notebook_r_enabled'] == 'true':
print("Installing R")
ensure_r(args.os_user, r_libs, args.region, args.r_mirror)
- print("Install Python 2 modules")
- ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
print("Install Python 3 specific version")
diff --git a/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/masters-user-data.sh b/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/masters-user-data.sh
index 58db632..6a6af5b 100644
--- a/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/masters-user-data.sh
+++ b/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/masters-user-data.sh
@@ -44,7 +44,7 @@
sudo chmod 600 /home/${k8s_os_user}/.ssh/authorized_keys
sudo apt-get update
-sudo apt-get install -y python-pip jq unzip
+sudo apt-get install -y python3-pip jq unzip
sudo apt-get install -y default-jre
sudo apt-get install -y default-jdk
sudo pip install -U pip
diff --git a/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/workers-user-data.sh b/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/workers-user-data.sh
index 8ff7695..a53bf03 100644
--- a/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/workers-user-data.sh
+++ b/infrastructure-provisioning/terraform/aws/ssn-k8s/main/files/workers-user-data.sh
@@ -37,7 +37,7 @@
sudo chmod 600 /home/${k8s_os_user}/.ssh/authorized_keys
sudo apt-get update
-sudo apt-get install -y python-pip
+sudo apt-get install -y python3-pip
sudo pip install -U pip
sudo pip install awscli
diff --git a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
index 7b188ac..1f179c5 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
+++ b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
@@ -276,7 +276,7 @@
conn.sudo('apt-get update')
conn.sudo('apt-get -y --allow-unauthenticated install mongodb-org')
conn.sudo('systemctl enable mongod.service')
- conn.sudo('sudo apt-get -y install python-pip')
+ conn.sudo('sudo apt-get -y install python3-pip')
conn.sudo('pip install -U pymongo pyyaml --no-cache-dir ')
conn.sudo('touch /home/{}/.ensure_dir/mongo_ensured'
.format(args.os_user))
@@ -292,7 +292,7 @@
conn.sudo('sed -i "s|PASSWORD|{}|g" /tmp/configure_mongo.py'.format(args.mongo_password))
if not exists(conn, '/tmp/mongo_roles.json'):
conn.put('./mongo_files/gcp/mongo_roles.json', '/tmp/mongo_roles.json')
- conn.sudo('python /tmp/configure_mongo.py')
+ conn.sudo('python3 /tmp/configure_mongo.py')
except Exception as err:
logging.error('Failed to install Mongo: ', str(err))
traceback.print_exc()
@@ -985,7 +985,7 @@
parser.add_argument('--repository_user', type=str, default='')
parser.add_argument('--repository_pass', type=str, default='')
parser.add_argument('--docker_version', type=str,
- default='18.06.3~ce~3-0~ubuntu')
+ default='5:20.10.2~3-0~ubuntu-focal')
parser.add_argument('--ssn_bucket_name', type=str, default='')
parser.add_argument('--keycloak_auth_server_url', type=str, default='')
parser.add_argument('--keycloak_realm_name', type=str, default='')
diff --git a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
index 96a615b..acaf6ef 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
+++ b/infrastructure-provisioning/terraform/bin/deploy/mongo_files/configure_mongo.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
index c513b10..28aafd3 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
@@ -494,7 +494,7 @@
parser.add_argument('--repository_user', type=str, default='')
parser.add_argument('--repository_pass', type=str, default='')
parser.add_argument('--docker_version', type=str,
- default='18.06.3~ce~3-0~ubuntu')
+ default='5:20.10.2~3-0~ubuntu-focal')
parser.add_argument('--ssn_bucket_name', type=str, default='')
parser.add_argument('--endpoint_keystore_password', type=str, default='')
parser.add_argument('--keycloak_client_secret', type=str, default='')
diff --git a/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md b/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md
index 82fdab2..8e8766f 100644
--- a/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md
+++ b/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md
@@ -34,7 +34,7 @@
```
add-apt-repository ppa:deadsnakes/ppa
apt-get install python3.7
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2
update-alternatives --config python3
apt-get install python3-pip
diff --git a/integration-tests/examples/copy_files.py b/integration-tests/examples/copy_files.py
index f96398d..7e5ed4f 100644
--- a/integration-tests/examples/copy_files.py
+++ b/integration-tests/examples/copy_files.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/integration-tests/examples/scenario_deeplearning/deeplearning_tests.py b/integration-tests/examples/scenario_deeplearning/deeplearning_tests.py
index 92a9596..4978328 100644
--- a/integration-tests/examples/scenario_deeplearning/deeplearning_tests.py
+++ b/integration-tests/examples/scenario_deeplearning/deeplearning_tests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/integration-tests/examples/scenario_jupyter/jupyter_tests.py b/integration-tests/examples/scenario_jupyter/jupyter_tests.py
index 018e678..1fe4354 100644
--- a/integration-tests/examples/scenario_jupyter/jupyter_tests.py
+++ b/integration-tests/examples/scenario_jupyter/jupyter_tests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/integration-tests/examples/scenario_rstudio/rstudio_tests.py b/integration-tests/examples/scenario_rstudio/rstudio_tests.py
index d9a1540..bd9764b 100644
--- a/integration-tests/examples/scenario_rstudio/rstudio_tests.py
+++ b/integration-tests/examples/scenario_rstudio/rstudio_tests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/integration-tests/examples/scenario_tensor/tensor_tests.py b/integration-tests/examples/scenario_tensor/tensor_tests.py
index 3bbb93b..a9985c0 100644
--- a/integration-tests/examples/scenario_tensor/tensor_tests.py
+++ b/integration-tests/examples/scenario_tensor/tensor_tests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/integration-tests/examples/scenario_zeppelin/zeppelin_tests.py b/integration-tests/examples/scenario_zeppelin/zeppelin_tests.py
index 1c9f05c..daa1c6b 100644
--- a/integration-tests/examples/scenario_zeppelin/zeppelin_tests.py
+++ b/integration-tests/examples/scenario_zeppelin/zeppelin_tests.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
# *****************************************************************************
#
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java
index 70e9a9e..72485e0 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java
@@ -94,6 +94,5 @@
super.setMasterGPUType(masterGPUType);
super.setSlaveGPUType(slaveGPUType);
super.setEnabledGPU(enabledGPU);
-
}
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.html
index c031cfa..b39ecd4 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.html
@@ -28,7 +28,6 @@
<div class="form-wrapper" [ngClass]="{ compress: selectedImage?.image === 'docker.datalab-dataengine' }">
<div class="col">
-
<div class="control-group" *ngIf="PROVIDER !== 'azure'" [hidden]="clusterTypes.length === 1">
<label class="label">Select cluster type</label>
<div class="control selector-wrapper">
@@ -111,7 +110,7 @@
<mat-select formControlName="shape_master" disableOptionCentering>
<mat-optgroup *ngFor="let item of (selectedImage.computation_resources_shapes | keys)"
[label]="item.key | underscoreless">
- <mat-option *ngFor="let list_item of item.value" [value]="list_item.Type">
+ <mat-option *ngFor="let list_item of item.value" [value]="list_item.Type" (click)="clearGpuType('master')">
<strong class="highlight icon-label">{{ list_item.Size }}</strong> {{ list_item.Type }}
</mat-option>
</mat-optgroup>
@@ -132,7 +131,7 @@
<mat-select formControlName="shape_slave" disableOptionCentering>
<mat-optgroup *ngFor="let item of (selectedImage.computation_resources_shapes | keys)"
[label]="item.key | underscoreless">
- <mat-option *ngFor="let list_item of item.value" [value]="list_item.Type">
+ <mat-option *ngFor="let list_item of item.value" [value]="list_item.Type" (click)="clearGpuType('slave')">
<strong class="highlight icon-label">{{ list_item.Size }}</strong> {{
list_item.Type }}
</mat-option>
@@ -147,21 +146,136 @@
</div>
</div>
+ <div class="checkbox-group control-group"
+ *ngIf="PROVIDER === 'gcp' && selectedImage?.image === 'docker.datalab-dataengine-service'">
+ <div class="d-flex cursor-pointer label m-bott-20" (click)="addAdditionalParams('gpu')">
+ <div class="empty-checkbox ml-10" [ngClass]="{'checked': isSelected.gpu}">
+ <span class="checked-checkbox" *ngIf="isSelected.gpu"></span>
+ </div>
+ <span class=" pl-5">GPU</span>
+ </div>
+ <div class="form-wrapper full-width" *ngIf="isSelected.gpu">
+ <div class="col">
+ <div class="control-group">
+ <label class="label">Master GPU type</label>
+ <div class="control selector-wrapper"
+ [ngClass]="{ 'not-active' : !resourceForm.controls['shape_master'].value}"
+ [matTooltip]="'Please, select master instance size.'"
+ [matTooltipPosition]="'above'"
+ [matTooltipClass]="'full-size-tooltip'"
+ [matTooltipDisabled]="!!resourceForm.controls['shape_master'].value.length"
+ >
+ <mat-form-field>
+ <mat-select formControlName="master_GPU_type" disableOptionCentering
+ placeholder="Select master GPU type"
+ [disabled]="!resourceForm.controls['shape_master'].value">
+ <mat-option *ngFor="let type of selectedImage.gpu_types"
+ [value]="type.Gpu_type" (click)="setCount('master', type.Gpu_type)">
+ <strong class="highlight icon-label">{{ type.Size }}</strong> {{ type.Gpu_type }}
+ </mat-option>
+ <mat-option *ngIf="!clusterTypes.length" class="multiple-select ml-10" disabled>Master GPU types list
+ is empty</mat-option>
+ </mat-select>
+ <button class="caret">
+ <i class="material-icons">keyboard_arrow_down</i>
+ </button>
+ </mat-form-field>
+ </div>
+ </div>
+ <div class="control-group">
+ <label class="label">Master GPU сount</label>
+ <div class="control selector-wrapper"
+ [ngClass]="{'not-active': !resourceForm.controls['master_GPU_type'].value}"
+ [matTooltip]="'Please, select master GPU type.'"
+ [matTooltipPosition]="'above'"
+ [matTooltipClass]="'full-size-tooltip'"
+ [matTooltipDisabled]="!!resourceForm.controls['master_GPU_type'].value"
+ >
+ <mat-form-field>
+ <mat-label>Select master GPU type</mat-label>
+ <mat-select formControlName="master_GPU_count" disableOptionCentering [disabled]="!resourceForm.controls['master_GPU_type'].value">
+ <mat-option *ngFor="let type of masterGPUcount" [value]="type">
+ {{ type }}
+ </mat-option>
+ <mat-option *ngIf="!masterGPUcount?.length" class="multiple-select ml-10" disabled>Slave GPU types list is empty</mat-option>
+ </mat-select>
+ <button class="caret">
+ <i class="material-icons">keyboard_arrow_down</i>
+ </button>
+ </mat-form-field>
+ </div>
+ </div>
+ </div>
- <div class="preemptible checkbox-group control-group m-top-30 m-bott-10"
+ <div class="col">
+ <div class="control-group">
+ <label class="label">Slave GPU type</label>
+ <div class="control selector-wrapper"
+ [ngClass]="{ 'not-active': !resourceForm.controls['shape_slave'].value}"
+ [matTooltip]="'Please, select slave instance size.'"
+ [matTooltipPosition]="'above'"
+ [matTooltipClass]="'full-size-tooltip'"
+ [matTooltipDisabled]="!!resourceForm.controls['shape_slave'].value.length"
+ >
+ <!-- <span class="form-field-wrapper" >-->
+ <mat-form-field>
+ <mat-label>Select slave GPU type</mat-label>
+ <mat-select formControlName="slave_GPU_type" disableOptionCentering [disabled]="!resourceForm.controls['shape_slave'].value">
+ <mat-option *ngFor="let type of selectedImage.gpu_types" [value]="type.Gpu_type" (click)="setCount('slave', type.Gpu_type)">
+ <strong class="highlight icon-label">{{ type.Size }}</strong> {{ type.Gpu_type }}
+ </mat-option>
+ <mat-option *ngIf="!selectedImage.gpu_types?.length" class="multiple-select ml-10" disabled>Slave GPU types list is empty</mat-option>
+ </mat-select>
+ <button class="caret">
+ <i class="material-icons">keyboard_arrow_down</i>
+ </button>
+ </mat-form-field>
+ <!-- </span>-->
+ </div>
+ </div>
+
+ <div class="control-group">
+ <label class="label">Slave GPU сount</label>
+ <div class="control selector-wrapper"
+ [ngClass]="{'not-active': !resourceForm.controls['slave_GPU_type'].value}"
+ [matTooltip]="'Please, select slave GPU type.'"
+ [matTooltipPosition]="'above'"
+ [matTooltipClass]="'full-size-tooltip'"
+ [matTooltipDisabled]="!!resourceForm.controls['slave_GPU_type'].value"
+ >
+ <mat-form-field>
+ <mat-label>Select slave GPU сount</mat-label>
+ <mat-select formControlName="slave_GPU_count" disableOptionCentering [disabled]="!resourceForm.controls['slave_GPU_type'].value">
+ <mat-option *ngFor="let type of slaveGPUcount" [value]="type">
+ {{ type }}
+ </mat-option>
+ <mat-option *ngIf="!slaveGPUcount?.length" class="multiple-select ml-10" disabled>Slave GPU types list is empty</mat-option>
+ </mat-select>
+ <button class="caret">
+ <i class="material-icons">keyboard_arrow_down</i>
+ </button>
+ </mat-form-field>
+ </div>
+ </div>
+
+ </div>
+ </div>
+ </div>
+ <div class="preemptible checkbox-group control-group"
*ngIf="PROVIDER === 'gcp' && selectedImage?.image === 'docker.datalab-dataengine-service'">
- <label class="label">
- <input #preemptibleNode type="checkbox" (change)="selectPreemptibleNodes($event)"/>
- <span>Preemptible node</span>
- <span class="align" [hidden]="!preemptible?.nativeElement['checked']"> count</span>
- </label>
- <div *ngIf="preemptible?.nativeElement['checked']" class="preemptible-details control"
- [ngClass]="{ show: preemptible?.nativeElement['checked'] || false}">
+ <div class="d-flex cursor-pointer label" (click)="addAdditionalParams('preemptible')">
+ <div class="empty-checkbox ml-10" [ngClass]="{'checked': isSelected.preemptible}" (click)="selectPreemptibleNodes(isSelected.preemptible)">
+ <span class="checked-checkbox" *ngIf="isSelected.preemptible"></span>
+ </div>
+ <span class="pl-5">Preemptible node</span>
+ </div>
+ <div *ngIf="isSelected.preemptible" class="preemptible-details control"
+ [ngClass]="{ show: isSelected.preemptible}">
<input type="text" class="form-control" formControlName="preemptible_instance_number"
(keypress)="CheckUtils.isNumberKey($event)"
(keydown.arrowup)="preemptibleCounter($event, 'increment')"
(keydown.arrowdown)="preemptibleCounter($event, 'decrement')"/>
- <span class="error" *ngIf="!resourceForm?.controls.preemptible_instance_number.valid">
+ <span class="error error-bottom" *ngIf="!resourceForm?.controls.preemptible_instance_number.valid">
<span *ngIf="minPreemptibleInstanceNumber !== maxPreemptibleInstanceNumber; else equal">
Only integer values greater than or equal to {{ minPreemptibleInstanceNumber }} and less than
{{ maxPreemptibleInstanceNumber }} are allowed.
@@ -169,36 +283,38 @@
<ng-template #equal>Please manage total machines count.</ng-template>
</span>
</div>
- </div>
+ </div>
- <div class="checkbox-group control-group m-top-15" *ngIf="PROVIDER === 'aws'"
- [hidden]="!selectedImage.templates.length">
- <label class="spot-label label">
- <input #spotInstancesCheck type="checkbox" (change)="selectSpotInstances($event)" />
- <span>Spot instance</span>
- <span *ngIf="spotInstancesSelect?.nativeElement['checked'] || false"> bid, %</span>
- </label>
- <div class="control spot-details" [ngClass]="{ show: spotInstancesSelect?.nativeElement['checked'] || false }"
- *ngIf="spotInstancesSelect?.nativeElement['checked'] || false">
+
+ <div class="checkbox-group control-group m-top-15" *ngIf="PROVIDER === 'aws'" [hidden]="!selectedImage.templates.length">
+ <div class="d-flex cursor-pointer label" (click)="addAdditionalParams('spotInstances')">
+ <div class="empty-checkbox ml-10" [ngClass]="{'checked': isSelected.spotInstances}" (click)="selectSpotInstances()">
+ <span class="checked-checkbox" *ngIf="isSelected.spotInstances"></span>
+ </div>
+ <span class="pl-5">Spot instance</span><span [hidden]="!isSelected.spotInstances"> bid, %</span>
+ </div>
+ <div class="control spot-details" [ngClass]="{ show: isSelected.spotInstances }"
+ *ngIf="isSelected.spotInstances">
<input type="number" class="form-control" step="5" min="{{minSpotPrice}}" max="{{maxSpotPrice}}"
formControlName="instance_price" (keypress)="CheckUtils.isNumberKey($event)">
- <span class="error" *ngIf="!resourceForm?.controls.instance_price.valid">
+ <span class="error error-bottom" *ngIf="!resourceForm?.controls.instance_price.valid">
Only integer values greater than or equal to {{minSpotPrice}} and less than {{maxSpotPrice}} are allowed.
</span>
</div>
- <span class="info" *ngIf="spotInstancesSelect?.nativeElement['checked'] || false">When the current Spot price
+ <span class="info ml-10" *ngIf="isSelected.spotInstances">When the current Spot price
rises above your bid price, the Spot instance is reclaimed by AWS so that it can be given to another
customer. Make sure to backup your data on periodic basis.</span>
</div>
-
-
- <div class="checkbox-group m-top-20"
+ <div class="checkbox-group m-top-20 control-group"
[hidden]="PROVIDER === 'gcp' && selectedImage?.image === 'docker.datalab-dataengine-service'"
*ngIf="notebook_instance?.image !== 'docker.datalab-zeppelin'">
- <label>
- <input #configurationNode type="checkbox" (change)="selectConfiguration()"/> Cluster configurations
- </label>
- <div class="config-link" *ngIf="(configuration?.nativeElement['checked'] || false)
+ <div class="d-flex cursor-pointer label" (click)="addAdditionalParams('configuration')">
+ <div class="empty-checkbox ml-10" [ngClass]="{'checked': isSelected.configuration}" (click)="selectConfiguration()">
+ <span class="checked-checkbox" *ngIf="isSelected.configuration"></span>
+ </div>
+ <span class="pl-5">Cluster configurations</span>
+ </div>
+ <div class="config-link" *ngIf="(isSelected.configuration)
&& selectedImage?.image === 'docker.datalab-dataengine-service'
&& PROVIDER === 'aws'">
To view example JSON of configurations refer for <a
@@ -207,8 +323,8 @@
official documentation</a>
</div>
</div>
- <div class="checkbox-group">
- <div class="config-details" [ngClass]="{ show: configuration?.nativeElement['checked'] || false }">
+ <div class="checkbox-group ml-10">
+ <div class="config-details" [ngClass]="{ show: isSelected.configuration }">
<textarea formControlName="configuration_parameters" placeholder="Cluster configuration template, JSON"
data-gramm_editor="false"></textarea>
<span class="error"
@@ -216,7 +332,7 @@
parameters is not in a valid format.</span>
</div>
</div>
- <div *ngIf="notebook_instance?.image === 'docker.datalab-zeppelin'">
+ <div *ngIf="notebook_instance?.image === 'docker.datalab-zeppelin'" class="ml-10">
<small>Spark default configuration for Apache Zeppelin can not be changed from DataLab UI. Currently it
can be
done directly through Apache Zeppelin interpreter menu.
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.scss b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.scss
index b8cd880..c849fc8 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.scss
@@ -74,9 +74,11 @@
opacity: 1;
}
}
- .preemptible {
+ &.preemptible {
+ height: 50px;
.label {
- padding-left: 10px;
+ height: 36px;
+ padding-bottom: 14px;
}
.control {
position: relative;
@@ -88,7 +90,7 @@
.spot-details, .preemptible-details {
position: relative;
.error {
- bottom: -5px;
+ bottom: 0;
}
&.show {
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.ts
index 8d42285..4822dc0 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create-dialog.component.ts
@@ -28,6 +28,7 @@
import { DICTIONARY } from '../../../../dictionary/global.dictionary';
import { CLUSTER_CONFIGURATION } from './cluster-configuration-templates';
+import {Logger} from 'codelyzer/util/logger';
@Component({
selector: 'computational-resource-create-dialog',
@@ -58,10 +59,14 @@
public minSpotPrice: number = 0;
public maxSpotPrice: number = 0;
public resourceForm: FormGroup;
-
- @ViewChild('spotInstancesCheck') spotInstancesSelect;
- @ViewChild('preemptibleNode') preemptible;
- @ViewChild('configurationNode') configuration;
+ public masterGPUcount: Array<number>;
+ public slaveGPUcount: Array<number>;
+ public isSelected = {
+ preemptible: false,
+ gpu: false,
+ spotInstances: false,
+ configuration: false,
+ };
constructor(
@Inject(MAT_DIALOG_DATA) public data: any,
@@ -90,8 +95,8 @@
this.resourceForm.controls['version'].setValue($event.templates[0].version);
}
- public selectSpotInstances($event?): void {
- if ($event ? $event.target.checked : (this.spotInstancesSelect && this.spotInstancesSelect.nativeElement['checked'])) {
+ public selectSpotInstances(): void {
+ if (this.isSelected.spotInstances) {
this.spotInstance = true;
this.resourceForm.controls['instance_price'].setValue(50);
} else {
@@ -100,13 +105,14 @@
}
}
- public selectPreemptibleNodes($event) {
- if ($event.target.checked)
+ public selectPreemptibleNodes(addPreemptible) {
+ if (addPreemptible) {
this.resourceForm.controls['preemptible_instance_number'].setValue(this.minPreemptibleInstanceNumber);
+ }
}
public selectConfiguration() {
- if (this.configuration && this.configuration.nativeElement.checked) {
+ if (this.isSelected.configuration) {
const template = (this.selectedImage.image === 'docker.datalab-dataengine-service')
? CLUSTER_CONFIGURATION.EMR
: CLUSTER_CONFIGURATION.SPARK;
@@ -132,7 +138,8 @@
}
public createComputationalResource(data) {
- this.model.createComputationalResource(data, this.selectedImage, this.notebook_instance, this.spotInstance, this.PROVIDER.toLowerCase())
+ this.model.createComputationalResource(data, this.selectedImage, this.notebook_instance,
+ this.spotInstance, this.PROVIDER.toLowerCase(), this.isSelected.gpu)
.subscribe((response: any) => {
if (response.status === HTTP_STATUS_CODES.OK) this.dialogRef.close(true);
}, error => this.toastr.error(error.message, 'Oops!'));
@@ -155,7 +162,11 @@
this.validPreemptibleRange.bind(this)])],
instance_price: [0, [this.validInstanceSpotRange.bind(this)]],
configuration_parameters: ['', [this.validConfiguration.bind(this)]],
- custom_tag: [this.notebook_instance.tags.custom_tag]
+ custom_tag: [this.notebook_instance.tags.custom_tag],
+ master_GPU_type: [''],
+ slave_GPU_type: [''],
+ master_GPU_count: [''],
+ slave_GPU_count: [''],
});
}
@@ -179,7 +190,7 @@
this.minSpotPrice = this.selectedImage.limits.min_emr_spot_instance_bid_pct;
this.maxSpotPrice = this.selectedImage.limits.max_emr_spot_instance_bid_pct;
- if (this.spotInstancesSelect) this.spotInstancesSelect.nativeElement['checked'] = true;
+ this.isSelected.spotInstances = true;
this.selectSpotInstances();
}
@@ -189,6 +200,7 @@
}
// Validation
+
private validInstanceNumberRange(control) {
if (control && control.value)
if (this.PROVIDER === 'gcp' && this.selectedImage.image === 'docker.datalab-dataengine-service') {
@@ -200,8 +212,8 @@
}
private validPreemptibleRange(control) {
- if (this.preemptible)
- return this.preemptible.nativeElement['checked']
+ if (this.isSelected.preemptible)
+ return this.isSelected.preemptible
? (control.value !== null
&& control.value >= this.minPreemptibleInstanceNumber
&& control.value <= this.maxPreemptibleInstanceNumber ? null : { valid: false })
@@ -221,16 +233,16 @@
}
private validInstanceSpotRange(control) {
- if (this.spotInstancesSelect)
- return this.spotInstancesSelect.nativeElement['checked']
+ if (this.isSelected.spotInstances)
+ return this.isSelected.spotInstances
? (control.value >= this.minSpotPrice && control.value <= this.maxSpotPrice ? null : { valid: false })
: control.value;
}
private validConfiguration(control) {
- if (this.configuration)
- return this.configuration.nativeElement['checked']
- ? (control.value && control.value !== null && CheckUtils.isJSON(control.value) ? null : { valid: false })
+ if (this.isSelected.configuration)
+ return this.isSelected.configuration ?
+ (control.value && control.value !== null && CheckUtils.isJSON(control.value) ? null : { valid: false })
: null;
}
@@ -316,4 +328,65 @@
CheckUtils.delimitersFiltering(conputational_resource_name) === CheckUtils.delimitersFiltering(resource));
}
}
+
+ public addAdditionalParams(block: string) {
+ this.isSelected[block] = !this.isSelected[block];
+ if (block === 'gpu') {
+ const controls = ['master_GPU_type', 'master_GPU_count', 'slave_GPU_type', 'slave_GPU_count'];
+ if (!this.isSelected.gpu) {
+ this.clearGpuType('master');
+ this.clearGpuType('slave');
+ controls.forEach(control => {
+ this.resourceForm.controls[control].clearValidators();
+ this.resourceForm.controls[control].updateValueAndValidity();
+ });
+
+ } else {
+ controls.forEach(control => {
+ this.resourceForm.controls[control].setValidators([Validators.required]);
+ this.resourceForm.controls[control].updateValueAndValidity();
+ });
+ }
+ }
+ }
+
+ public setGPUCount(type, gpuType): Array<number> {
+ let count = [];
+ switch (type) {
+ case 'n1-highmem-32' || 'n1-highcpu-32':
+ count = [4, 8];
+ break;
+ case 'n1-highmem-16':
+ count = [2, 4, 8];
+ break;
+ default:
+ count = [1, 2, 4, 8];
+ break;
+ }
+ if (gpuType === 'nvidia-tesla-t4') {
+ count.pop();
+ }
+
+ return count;
+ }
+
+ public setCount(type: any, gpuType: any): void {
+ if (type === 'master') {
+ const masterShape = this.resourceForm.controls['shape_master'].value;
+ this.masterGPUcount = this.setGPUCount(masterShape, gpuType);
+ } else {
+ const slaveShape = this.resourceForm.controls['shape_slave'].value;
+ this.slaveGPUcount = this.setGPUCount(slaveShape, gpuType);
+ }
+ }
+
+ public clearGpuType(type) {
+ if (type === 'master') {
+ this.resourceForm.controls['master_GPU_type'].setValue('');
+ this.resourceForm.controls['master_GPU_count'].setValue('');
+ } else {
+ this.resourceForm.controls['slave_GPU_type'].setValue('');
+ this.resourceForm.controls['slave_GPU_count'].setValue('');
+ }
+ }
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create.model.ts b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create.model.ts
index 2f6a655..93eff8d 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create.model.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resource-create-dialog/computational-resource-create.model.ts
@@ -28,7 +28,7 @@
constructor(private userResourceService: UserResourceService) { }
- public createComputationalResource(parameters, image, env, spot, provider): Observable<{}> {
+ public createComputationalResource(parameters, image, env, spot, provider, gpu?): Observable<{}> {
const config = parameters.configuration_parameters ? JSON.parse(parameters.configuration_parameters) : null;
if (provider === 'aws' && image.image === 'docker.datalab-dataengine-service') {
@@ -61,7 +61,12 @@
dataproc_version: parameters.version,
config: config,
project: env.project,
- custom_tag: parameters.custom_tag
+ custom_tag: parameters.custom_tag,
+ masterGPUType: gpu ? parameters.master_GPU_type : null,
+ slaveGPUType: gpu ? parameters.slave_GPU_type : null,
+ masterGPUCount: gpu ? parameters.master_GPU_count : null,
+ slaveGPUCount: gpu ? parameters.slave_GPU_count : null,
+ gpu_enabled: gpu
}, provider);
} else {
return this.userResourceService.createComputationalResource_Dataengine({
diff --git a/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss b/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss
index c8e952a..8f1f86e 100644
--- a/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss
+++ b/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss
@@ -48,7 +48,7 @@
}
.modal-xxl {
- width: 1005px;
+ width: 1100px;
}
.modal-fullscreen {
diff --git a/services/self-service/src/main/resources/webapp/src/assets/styles/_general.scss b/services/self-service/src/main/resources/webapp/src/assets/styles/_general.scss
index e27d33f..16ffc1e 100644
--- a/services/self-service/src/main/resources/webapp/src/assets/styles/_general.scss
+++ b/services/self-service/src/main/resources/webapp/src/assets/styles/_general.scss
@@ -42,6 +42,8 @@
.pb-25 {padding-bottom: 25px;}
.pb-10 {padding-bottom: 10px;}
+.full-width{width: 100%;}
+
.txt-r {text-align: right }
.txt-l {text-align: left }
@@ -215,4 +217,8 @@
right: -15px !important;
}
+.cursor-pointer{
+ cursor: pointer;
+}
+
diff --git a/services/self-service/src/main/resources/webapp/src/styles.scss b/services/self-service/src/main/resources/webapp/src/styles.scss
index 3c5d948..27b6668 100644
--- a/services/self-service/src/main/resources/webapp/src/styles.scss
+++ b/services/self-service/src/main/resources/webapp/src/styles.scss
@@ -311,6 +311,10 @@
padding-left: 5px;
}
+.pl-10{
+ padding-left: 10px !important;
+}
+
.pl-20{
padding-left: 20px;
}
