dev-support/submarine-installer/install.conf - submarine - Git at Google

 #!/usr/bin/env bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 ################################ DOWNLOAD CONFIGURATION ####################################################
 # Start the http download service on the specified server,
 # Will download all the dependencies in the http server,
 # Run the install script on other servers.
 # Automatically download dependencies from http,
 # Solve the problem that all servers are slow to download online.
 # At the same time, you can also manually download the dependencies to the downloads directory.
 # Offline installation of the system
 # The host ip on which download service is started.
 DOWNLOAD_SERVER_IP="10.196.69.175"
 DOWNLOAD_SERVER_PORT="19000"

 ################################ DNS CONFIGURATION #################################################
 # The default DNS ip for the cluster
 LOCAL_DNS_HOST="172.17.0.9"   # /etc/resolv.conf
 # see YARN_REGISTRY_DNS_HOST

 # determine whether install an security yarn ,value is "true" or "false",default is "true"
 YARN_SECURITY="true"

 ################################# YARN CONFIGURATION #############################################
 # The resourcemanager hosts. For example, YARN_RESOURCE_MANAGER_HOSTS=(hostname1 hostname2)
 YARN_RESOURCE_MANAGER_HOSTS=()
 # The hosts should not install nodemanager. Multiple hosts should be split by spaces. For example, YARN_NODE_MANAGER_EXCLUDE_HOSTS=(hostname1 hostname2)
 YARN_NODE_MANAGER_EXCLUDE_HOSTS=()
 # hadoop.registry.dns.bind-port & hadoop.registry.dns.bind-address in yarn-site.xml
 YARN_REGISTRY_DNS_HOST="" # yarn registry dns host
 YARN_REGISTRY_DNS_HOST_PORT="53"
 YARN_TIMELINE_HOST="" # timeline-v1.5 & v2.0
 YARN_JOB_HISTORY_HOST=""
 YARN_SPARK_HISTORY_HOST=""

 # Kerberos realm used in core-site.xml
 LOCAL_REALM=""
 # The namenode hosts. For example, HDFS_NAMENODE_HOSTS=(hostname1 hostname2)
 HDFS_NAMENODE_HOSTS=()

 # fs.defaultFS in core-site.xml. For example, FS_DEFAULTFS="hdfs://machine-learning"
 FS_DEFAULTFS=""

 # local.cluster-id in yarn-site.xml
 LOCAL_CLUSTER_ID="ml-yarn"

 # yarn.app.mapreduce.am.staging-dir in mapred-site.xml
 YARN_APP_MAPREDUCE_AM_STAGING_DIR="/tmp/ml-yarn-staging"

 # yarn container-executor config
 # How to compile container-executor:
 # Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
 # path and enter the command: `mvn package -Pnative -DskipTests`
 # Only the nodemanager is compiled. The path of the compiled container-executor is:
 # ./target/native/target/usr/local/bin/container-executor
 YARN_CONTAINER_EXECUTOR_PATH="/etc/yarn/sbin/Linux-amd64-64/container-executor"

 # Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy'
 # in yarn-site.xml, default '/hadoop-yarn'
 YARN_HIERARCHY="/hadoop-yarn"

 # The home paths for container dir
 YARN_NODEMANAGER_LOCAL_HOME_PATHS="/home/hadoop/disk/1/yarn,/home/hadoop/disk/2/yarn,/home/hadoop/disk/3/yarn"

 # Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml
 YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local"

 # Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml
 YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs"

 # Create '/app-logs' path on hdfs, Owner is 'yarn', group is 'hadoop',
 # and 'hadoop' group needs to include 'hdfs, yarn, mapred' yarn-site.xml users, etc.
 # `yarn.nodemanager.remote-app-log-dir` in yarn-site.xml
 YARN_AGGREGATED_LOG_DIR="/ml-app-logs"

 # time line v1.5 store path on hdfs
 YARN_TIMELINE_FS_STORE_DIR="/tmp/ml-entity-file-history"

 # hbase client
 YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE="file:/home/hadoop/hbase-current/conf/hbase-site.xml"

 # yarn.resourcemanager.zk-address in yarn-site.xml
 # hadoop.registry.zk.quorum in core-site.xml. For Example, YARN_ZK_ADDRESS="zookeeperHost1:2181,zookeeperHost2:2181,zookeeperHost3:2181"
 YARN_ZK_ADDRESS=""

 # yarn.timeline-service.leveldb-state-store.path in yarn-site.xml
 YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH="/home/hadoop/timeline"

 # yarn.nodemanager.recovery.dir in yarn-site.xml
 YARN_NODEMANAGER_RECOVERY_DIR="/home/hadoop/yarn/yarn-nm-recovery"

 # yarn.nodemanager.runtime.linux.docker.default-container-network in yarn-site.xml
 YARN_DOCKER_CONTAINER_DEFAULT_NETWORK=calico-network

 # docker.allowed-container-networks in yarn-site.xml and container-executor.cfg
 YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS=host,none,bridge,calico-network

 YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH="/home/hadoop/hosts/yarn_exclude"

 YARN_GC_LOG_DIR="/home/hadoop/yarn/gclog"
 YARN_PID_DIR="/home/hadoop/yarn/pid"
 YARN_LOG_DIR="/home/hadoop/yarn/log"

 ################################### YARN-kerberos CONFIGURATION ##############################################
 # Kerberos authentication to create some necessary hdfs paths
 HADOOP_KEYTAB_LOCATION=""
 HADOOP_PRINCIPAL=""

 MAPRED_KEYTAB_LOCATION="/home/hadoop/yarn/conf/mapred.service.keytab"

 # yarn.timeline-service.keytab in yarn-site.xml
 YARN_KEYTAB_LOCATION="/home/hadoop/yarn/conf/yarn.service.keytab"

 HTTP_KEYTAB_LOCATION="/home/hadoop/yarn/conf/http.service.keytab"

 ################################### JAVA_HOME CONFIGURATION ##############################################
 JAVA_VERSION=jdk1.8.0_152
 JAVA_TARBALL=${JAVA_VERSION}.tgz
 JAVA_HOME=/home/hadoop/java-current

 ################################### YARN-hadoop CONFIGURATION ##############################################
 HADOOP_SETUP_USER="hadoop"
 HADOOP_VERSION="hadoop-3.3.0-SNAPSHOT"
 HADOOP_TARBALL="${HADOOP_VERSION}.tar.gz"
 HADOOP_HOME="/home/hadoop/hadoop-current"

 # hadoop.http.authentication.signature.secret.file in core-site.xml
 HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE="/home/hadoop/yarn/conf/http-secret"

 ################################# YARN-hbase CONFIGURATION ##############################################
 # copy timeline hbase jar to ${hbase_client]/lib path
 HBASE_HOME="/home/hadoop/hbase-current"

 ################################ ETCD CONFIGURATION ####################################################
 ## etcd host ip list
 ETCD_HOSTS=(10.196.69.201 10.196.69.173 10.196.69.174)

 ETCD_VERSION="v3.3.9"
 ETCD_TAR_GZ="etcd-${ETCD_VERSION}-linux-amd64.tar.gz"
 ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/${ETCD_TAR_GZ}"

 ################################## CALICO CONFIGURATION ##########################################
 CALICO_IPV4POOL_CIDR="192.20.0.0"
 CALICO_NETWORK_NAME="calico-network"
 CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico"
 CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam"
 CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl"

 ################################ DOCKER CONFIG SEGMENT ###################################################
 # docker registry ip:port
 DOCKER_REGISTRY="10.120.196.232:5000"
 # /var/lib/docker is used if DOCKER_STORE_PATH not specified
 DOCKER_STORE_PATH=

 ##### System component download url address, Generally do not need to be modified #####
 DOCKER_REPO="https://download.docker.com/linux/centos/7/x86_64/stable/Packages/"
 DOCKER_VERSION="docker-ce"
 DOCKER_VERSION_NUM="18.06.1.ce"
 DOCKER_ENGINE_RPM="${DOCKER_VERSION}-${DOCKER_VERSION_NUM}-3.el7.x86_64.rpm"

 ################################# NVIDIA CONFIGURATION #############################################
 NVIDIA_DETECT_URL="http://mirror.rackspace.com/elrepo/elrepo/el7/x86_64/RPMS"
 # NVIDIA_DRIVER_VERSION can be used by download server to decide which version of
 # nvidia driver should be download
 NVIDIA_DRIVER_VERSION=""
 NVIDIA_DOCKER_VERSION="2.0.3"
 NVIDIA_DOCKER_GIT_SNAPSHOT_URL="https://api.github.com/repos/nvidia/"
	#!/usr/bin/env bash
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	################################ DOWNLOAD CONFIGURATION ####################################################
	# Start the http download service on the specified server,
	# Will download all the dependencies in the http server,
	# Run the install script on other servers.
	# Automatically download dependencies from http,
	# Solve the problem that all servers are slow to download online.
	# At the same time, you can also manually download the dependencies to the downloads directory.
	# Offline installation of the system
	# The host ip on which download service is started.
	DOWNLOAD_SERVER_IP="10.196.69.175"
	DOWNLOAD_SERVER_PORT="19000"

	################################ DNS CONFIGURATION #################################################
	# The default DNS ip for the cluster
	LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf
	# see YARN_REGISTRY_DNS_HOST

	# determine whether install an security yarn ,value is "true" or "false",default is "true"
	YARN_SECURITY="true"

	################################# YARN CONFIGURATION #############################################
	# The resourcemanager hosts. For example, YARN_RESOURCE_MANAGER_HOSTS=(hostname1 hostname2)
	YARN_RESOURCE_MANAGER_HOSTS=()
	# The hosts should not install nodemanager. Multiple hosts should be split by spaces. For example, YARN_NODE_MANAGER_EXCLUDE_HOSTS=(hostname1 hostname2)
	YARN_NODE_MANAGER_EXCLUDE_HOSTS=()
	# hadoop.registry.dns.bind-port & hadoop.registry.dns.bind-address in yarn-site.xml
	YARN_REGISTRY_DNS_HOST="" # yarn registry dns host
	YARN_REGISTRY_DNS_HOST_PORT="53"
	YARN_TIMELINE_HOST="" # timeline-v1.5 & v2.0
	YARN_JOB_HISTORY_HOST=""
	YARN_SPARK_HISTORY_HOST=""

	# Kerberos realm used in core-site.xml
	LOCAL_REALM=""
	# The namenode hosts. For example, HDFS_NAMENODE_HOSTS=(hostname1 hostname2)
	HDFS_NAMENODE_HOSTS=()

	# fs.defaultFS in core-site.xml. For example, FS_DEFAULTFS="hdfs://machine-learning"
	FS_DEFAULTFS=""

	# local.cluster-id in yarn-site.xml
	LOCAL_CLUSTER_ID="ml-yarn"

	# yarn.app.mapreduce.am.staging-dir in mapred-site.xml
	YARN_APP_MAPREDUCE_AM_STAGING_DIR="/tmp/ml-yarn-staging"

	# yarn container-executor config
	# How to compile container-executor:
	# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
	# path and enter the command: `mvn package -Pnative -DskipTests`
	# Only the nodemanager is compiled. The path of the compiled container-executor is:
	# ./target/native/target/usr/local/bin/container-executor
	YARN_CONTAINER_EXECUTOR_PATH="/etc/yarn/sbin/Linux-amd64-64/container-executor"

	# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy'
	# in yarn-site.xml, default '/hadoop-yarn'
	YARN_HIERARCHY="/hadoop-yarn"

	# The home paths for container dir
	YARN_NODEMANAGER_LOCAL_HOME_PATHS="/home/hadoop/disk/1/yarn,/home/hadoop/disk/2/yarn,/home/hadoop/disk/3/yarn"

	# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml
	YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local"

	# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml
	YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs"

	# Create '/app-logs' path on hdfs, Owner is 'yarn', group is 'hadoop',
	# and 'hadoop' group needs to include 'hdfs, yarn, mapred' yarn-site.xml users, etc.
	# `yarn.nodemanager.remote-app-log-dir` in yarn-site.xml
	YARN_AGGREGATED_LOG_DIR="/ml-app-logs"

	# time line v1.5 store path on hdfs
	YARN_TIMELINE_FS_STORE_DIR="/tmp/ml-entity-file-history"

	# hbase client
	YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE="file:/home/hadoop/hbase-current/conf/hbase-site.xml"

	# yarn.resourcemanager.zk-address in yarn-site.xml
	# hadoop.registry.zk.quorum in core-site.xml. For Example, YARN_ZK_ADDRESS="zookeeperHost1:2181,zookeeperHost2:2181,zookeeperHost3:2181"
	YARN_ZK_ADDRESS=""

	# yarn.timeline-service.leveldb-state-store.path in yarn-site.xml
	YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH="/home/hadoop/timeline"

	# yarn.nodemanager.recovery.dir in yarn-site.xml
	YARN_NODEMANAGER_RECOVERY_DIR="/home/hadoop/yarn/yarn-nm-recovery"

	# yarn.nodemanager.runtime.linux.docker.default-container-network in yarn-site.xml
	YARN_DOCKER_CONTAINER_DEFAULT_NETWORK=calico-network

	# docker.allowed-container-networks in yarn-site.xml and container-executor.cfg
	YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS=host,none,bridge,calico-network

	YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH="/home/hadoop/hosts/yarn_exclude"

	YARN_GC_LOG_DIR="/home/hadoop/yarn/gclog"
	YARN_PID_DIR="/home/hadoop/yarn/pid"
	YARN_LOG_DIR="/home/hadoop/yarn/log"

	################################### YARN-kerberos CONFIGURATION ##############################################
	# Kerberos authentication to create some necessary hdfs paths
	HADOOP_KEYTAB_LOCATION=""
	HADOOP_PRINCIPAL=""

	MAPRED_KEYTAB_LOCATION="/home/hadoop/yarn/conf/mapred.service.keytab"

	# yarn.timeline-service.keytab in yarn-site.xml
	YARN_KEYTAB_LOCATION="/home/hadoop/yarn/conf/yarn.service.keytab"

	HTTP_KEYTAB_LOCATION="/home/hadoop/yarn/conf/http.service.keytab"

	################################### JAVA_HOME CONFIGURATION ##############################################
	JAVA_VERSION=jdk1.8.0_152
	JAVA_TARBALL=${JAVA_VERSION}.tgz
	JAVA_HOME=/home/hadoop/java-current

	################################### YARN-hadoop CONFIGURATION ##############################################
	HADOOP_SETUP_USER="hadoop"
	HADOOP_VERSION="hadoop-3.3.0-SNAPSHOT"
	HADOOP_TARBALL="${HADOOP_VERSION}.tar.gz"
	HADOOP_HOME="/home/hadoop/hadoop-current"

	# hadoop.http.authentication.signature.secret.file in core-site.xml
	HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE="/home/hadoop/yarn/conf/http-secret"

	################################# YARN-hbase CONFIGURATION ##############################################
	# copy timeline hbase jar to ${hbase_client]/lib path
	HBASE_HOME="/home/hadoop/hbase-current"

	################################ ETCD CONFIGURATION ####################################################
	## etcd host ip list
	ETCD_HOSTS=(10.196.69.201 10.196.69.173 10.196.69.174)

	ETCD_VERSION="v3.3.9"
	ETCD_TAR_GZ="etcd-${ETCD_VERSION}-linux-amd64.tar.gz"
	ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/${ETCD_TAR_GZ}"

	################################## CALICO CONFIGURATION ##########################################
	CALICO_IPV4POOL_CIDR="192.20.0.0"
	CALICO_NETWORK_NAME="calico-network"
	CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico"
	CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam"
	CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl"

	################################ DOCKER CONFIG SEGMENT ###################################################
	# docker registry ip:port
	DOCKER_REGISTRY="10.120.196.232:5000"
	# /var/lib/docker is used if DOCKER_STORE_PATH not specified
	DOCKER_STORE_PATH=

	##### System component download url address, Generally do not need to be modified #####
	DOCKER_REPO="https://download.docker.com/linux/centos/7/x86_64/stable/Packages/"
	DOCKER_VERSION="docker-ce"
	DOCKER_VERSION_NUM="18.06.1.ce"
	DOCKER_ENGINE_RPM="${DOCKER_VERSION}-${DOCKER_VERSION_NUM}-3.el7.x86_64.rpm"

	################################# NVIDIA CONFIGURATION #############################################
	NVIDIA_DETECT_URL="http://mirror.rackspace.com/elrepo/elrepo/el7/x86_64/RPMS"
	# NVIDIA_DRIVER_VERSION can be used by download server to decide which version of
	# nvidia driver should be download
	NVIDIA_DRIVER_VERSION=""
	NVIDIA_DOCKER_VERSION="2.0.3"
	NVIDIA_DOCKER_GIT_SNAPSHOT_URL="https://api.github.com/repos/nvidia/"