blob: fb579ad21e28faeccf941eaf9eb06cf4bcdef32c [file] [log] [blame]
#!/usr/bin/python
# *****************************************************************************
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# ******************************************************************************
from dlab.actions_lib import *
from dlab.common_lib import *
from dlab.notebook_lib import *
from dlab.fab import *
from fabric.api import *
from fabric.contrib.files import exists
import argparse
import json
import sys
import os
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--region', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
parser.add_argument('--rstudio_pass', type=str, default='')
parser.add_argument('--rstudio_version', type=str, default='')
parser.add_argument('--r_mirror', type=str, default='')
parser.add_argument('--ip_adress', type=str, default='')
parser.add_argument('--exploratory_name', type=str, default='')
parser.add_argument('--edge_ip', type=str, default='')
args = parser.parse_args()
spark_version = os.environ['notebook_spark_version']
hadoop_version = os.environ['notebook_hadoop_version']
tensorflow_version = os.environ['notebook_tensorflow_version']
nvidia_version = os.environ['notebook_nvidia_version']
theano_version = os.environ['notebook_theano_version']
keras_version = os.environ['notebook_keras_version']
cuda_version = os.environ['notebook_cuda_version']
cuda_file_name = os.environ['notebook_cuda_file_name']
cudnn_version = os.environ['notebook_cudnn_version']
cudnn_file_name = os.environ['notebook_cudnn_file_name']
if args.region == 'cn-north-1':
spark_link = "http://mirrors.hust.edu.cn/apache/spark/spark-" + spark_version + "/spark-" + spark_version + \
"-bin-hadoop" + hadoop_version + ".tgz"
else:
spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/spark-" + spark_version + \
"-bin-hadoop" + hadoop_version + ".tgz"
local_spark_path = '/opt/spark/'
jars_dir = '/opt/jars/'
templates_dir = '/root/templates/'
files_dir = '/root/files/'
gitlab_certfile = os.environ['conf_gitlab_certfile']
r_libs = ['R6', 'pbdZMQ', 'RCurl', 'devtools', 'reshape2', 'caTools', 'rJava', 'ggplot2', 'evaluate', 'formatR', 'yaml',
'Rcpp', 'rmarkdown', 'base64enc', 'tibble', 'tensorflow']
##############
# Run script #
##############
if __name__ == "__main__":
print("Configure connections")
env['connection_attempts'] = 100
env.key_filename = [args.keyfile]
env.host_string = args.os_user + '@' + args.hostname
# PREPARE DISK
print("Prepare .ensure directory")
try:
if not exists('/home/' + args.os_user + '/.ensure_dir'):
sudo('mkdir /home/' + args.os_user + '/.ensure_dir')
except:
sys.exit(1)
print("Mount additional volume")
prepare_disk(args.os_user)
# INSTALL LANGUAGES
print("Install Java")
ensure_jre_jdk(args.os_user)
print("Install R")
ensure_r(args.os_user, r_libs, args.region, args.r_mirror)
print("Install Python 2 modules")
ensure_python2_libraries(args.os_user)
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
# INSTALL TENSORFLOW AND OTHER DEEP LEARNING LIBRARIES
print("Install TensorFlow")
install_tensor(args.os_user, cuda_version, cuda_file_name,
cudnn_version, cudnn_file_name, tensorflow_version,
templates_dir, nvidia_version)
print("Install Theano")
install_theano(args.os_user, theano_version)
print("Installing Keras")
install_keras(args.os_user, keras_version)
# INSTALL RSTUDIO
print("Install RStudio")
install_rstudio(args.os_user, local_spark_path, args.rstudio_pass, args.rstudio_version)
# INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
print("Install local Spark")
ensure_local_spark(args.os_user, spark_link, spark_version, hadoop_version, local_spark_path )
print("Install storage jars")
ensure_local_jars(args.os_user, jars_dir)
print("Configure local Spark")
configure_local_spark(jars_dir, templates_dir)
# INSTALL UNGIT
print("Install nodejs")
install_nodejs(args.os_user)
print("Install Ungit")
install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
install_gitlab_cert(args.os_user, gitlab_certfile)
# INSTALL INACTIVITY CHECKER
print("Install inactivity checker")
install_inactivity_checker(args.os_user, args.ip_adress)
# POST INSTALLATION PROCESS
print("Updating pyOpenSSL library")
update_pyopenssl_lib(args.os_user)