conf/uno.conf - fluo-uno - Git at Google

 #! /usr/bin/env bash
 # From load-env.sh, variable UNO_HOME defaults to root directory of this Uno repo.

 # Versions
 # --------
 export HADOOP_VERSION=${HADOOP_VERSION:-3.3.4}
 export ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.8.0}
 export ACCUMULO_VERSION=${ACCUMULO_VERSION:-2.1.0}
 export FLUO_VERSION=${FLUO_VERSION:-1.2.0}
 export FLUO_YARN_VERSION=${FLUO_YARN_VERSION:-1.0.0}

 # Hashes (supports MD5, SHA-1, SHA-256, and SHA-512; MD5 requires md5sum)
 # --------------
 # Hashes below match default versions above. If you change a version above,
 # you must also change the hash below.
 HADOOP_HASH=$(grep -F hadoop:"${HADOOP_VERSION}": "${UNO_HOME:?}"/conf/checksums | cut -d : -f 3)
 ZOOKEEPER_HASH=$(grep -F zookeeper:"${ZOOKEEPER_VERSION}": "${UNO_HOME:?}"/conf/checksums | cut -d : -f 3)
 ACCUMULO_HASH=$(grep -F accumulo:"${ACCUMULO_VERSION}": "${UNO_HOME:?}"/conf/checksums | cut -d : -f 3)
 FLUO_HASH=037f89cd2bfdaf76a1368256c52de46d6b9a85c9c1bfc776ec4447d02c813fb2
 FLUO_YARN_HASH=c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff874b0
 export HADOOP_HASH ZOOKEEPER_HASH ACCUMULO_HASH FLUO_HASH FLUO_YARN_HASH

 # Network configuration
 # ---------------------
 # Hostname to use in configuration. Set to $(hostname) to use your hostname
 export UNO_HOST=${UNO_HOST:-localhost}

 # Download configuration
 # ----------------------
 # Directory where all depedency packages are downloaded
 export DOWNLOADS=${UNO_DOWNLOADS:-$UNO_HOME/downloads}
 # Tarball file names
 export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
 export HADOOP_TARBALL=hadoop-"$HADOOP_VERSION".tar.gz
 export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
 export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
 export ZOOKEEPER_TARBALL=apache-zookeeper-"$ZOOKEEPER_VERSION"-bin.tar.gz
 if [[ $ZOOKEEPER_VERSION =~ ^3[.][01234].*$ ]]; then
     export ZOOKEEPER_TARBALL=zookeeper-"$ZOOKEEPER_VERSION".tar.gz
 fi
 # Building Accumulo
 #------------------
 # If set, 'uno fetch' will build (instead of downloading) an Accumulo tarball
 # from that directory and copy it to the downloads directory.

 #export ACCUMULO_REPO=/path/to/accumulo

 # Comment out the following if block if you don't want to automatically detect
 # version from the pom.xml. This could be useful if you want to switch branches
 # in your workspace and don't want the detected version to change.
 if [[ -n "$ACCUMULO_REPO" ]]; then
   # Detect the version from the accumulo pom.xml in the workspace
   ACCUMULO_VERSION=$(xmllint --shell "$ACCUMULO_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
   export ACCUMULO_VERSION
   export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
 fi

 # Building Fluo
 # -------------
 # If set, 'uno fetch' will build (rather than download) a Fluo tarball
 # from that directory and copy it to the downloads directory.

 #export FLUO_REPO=/path/to/fluo

 # Comment out the following if block if you don't want to automatically detect
 # version from the pom.xml. This could be useful if you want to switch branches
 # in your workspace and don't want the detected version to change.
 if [[ -n "$FLUO_REPO" ]]; then
   # Detect the version from the accumulo pom.xml in the workspace
   FLUO_VERSION=$(xmllint --shell "$FLUO_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
   export FLUO_VERSION
   export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
 fi

 # Building Fluo YARN
 # ------------------
 # If set, 'uno fetch' will build (rather than download) a Fluo YARN tarball
 # from that directory and copy it to the downloads directory.

 #export FLUO_YARN_REPO=/path/to/fluo-yarn

 # Comment out the following if block if you don't want to automatically detect
 # version from the pom.xml. This could be useful if you want to switch branches
 # in your workspace and don't want the detected version to change.
 if [[ -n "$FLUO_YARN_REPO" ]]; then
   # Detect the version from the accumulo pom.xml in the workspace
   FLUO_YARN_VERSION=$(xmllint --shell "$FLUO_YARN_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
   export FLUO_YARN_VERSION
   export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
 fi

 # Version compatibility checks
 # ----------------------------
 # Apache built Accumulo 1.x only works with Hadoop 2.x.  Apache built Accumulo
 # 2.x only works with Hadoop 3.x.  However if building Accumulo 1.x, it can
 # work with Hadoop 3.x.  The following sanity checks fail fast when versions
 # are incompatible.  If the checks cause a problem, comment them out and
 # consider submitting an issue or PR.
 if [[ "${HADOOP_VERSION}" = 1.* ]]; then
    echo "ERROR from uno.conf : Hadoop 1.x is not supported" 1>&2
    exit 1
 fi

 if [[ -z $ACCUMULO_REPO && ${ACCUMULO_VERSION} =~ ^1[.][0-9][.][0-9]+$ && ! ${HADOOP_VERSION} = 2.* ]]; then
    echo "ERROR from uno.conf : When using Accumulo versions prior to 1.10.x, expect Hadoop 2.x not $HADOOP_VERSION" 1>&2
    exit 1
 fi

 if [[ "${ACCUMULO_VERSION}" = 2.* && ! "${HADOOP_VERSION}" = 3.* ]]; then
   echo "ERROR from uno.conf : When using Accumulo 2.x, expect Hadoop 3.x not $HADOOP_VERSION" 1>&2
   exit 1
 fi

 # Installation directories
 # ------------------------
 # Software installation directory
 export INSTALL=$UNO_HOME/install
 # Directory where all Fluo dependencies store their data
 export DATA_DIR=$INSTALL/data
 # Home directories
 export ZOOKEEPER_HOME=$INSTALL/apache-zookeeper-$ZOOKEEPER_VERSION-bin
 if [[ $ZOOKEEPER_VERSION =~ ^3[.][01234].*$ ]]; then
   export ZOOKEEPER_HOME=$INSTALL/zookeeper-$ZOOKEEPER_VERSION
 fi
 export HADOOP_HOME=$INSTALL/hadoop-$HADOOP_VERSION
 export ACCUMULO_HOME=$INSTALL/accumulo-$ACCUMULO_VERSION
 export FLUO_HOME=$INSTALL/fluo-$FLUO_VERSION
 export FLUO_YARN_HOME=$INSTALL/fluo-yarn-$FLUO_YARN_VERSION
 # Config directories
 export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
 # Log directories
 export LOGS_DIR=$INSTALL/logs
 export ACCUMULO_LOG_DIR=$LOGS_DIR/accumulo
 export HADOOP_LOG_DIR=$LOGS_DIR/hadoop
 export ZOO_LOG_DIR=$LOGS_DIR/zookeeper

 # Accumulo configuration
 # ----------------------
 # Accumulo instance name
 export ACCUMULO_INSTANCE=uno
 # Accumulo user
 export ACCUMULO_USER=root
 # Accumulo password
 export ACCUMULO_PASSWORD=secret
 # Number of tservers to run
 export NUM_TSERVERS=1

 # Plugin configuration
 # --------------------
 # Post-install plugins. Example: "influx-metrics accumulo-encryption"
 export POST_INSTALL_PLUGINS=""
 # Post-run plugins. Example: "spark"
 export POST_RUN_PLUGINS=""
 # Configuration for 'spark' plugin
 export SPARK_VERSION=${SPARK_VERSION:-2.3.2}
 export SPARK_HOME=$INSTALL/spark-${SPARK_VERSION}-bin-without-hadoop
 export SPARK_TARBALL=spark-${SPARK_VERSION}-bin-without-hadoop.tgz
 SPARK_HASH=$(grep -F spark:"${SPARK_VERSION}": "${UNO_HOME:?}"/conf/checksums | cut -d : -f 3)
 export SPARK_HASH
 # Configuration for 'influxdb-metrics' plugin
 # InfluxDB metrics can only be set up on Linux. Mac OS X is not supported.
 export INFLUXDB_VERSION=0.9.4.2
 export INFLUXDB_HOME=$INSTALL/influxdb-"$INFLUXDB_VERSION"
 export INFLUXDB_TARBALL=influxdb-"$INFLUXDB_VERSION".tar.gz
 export INFLUXDB_HASH=fe4269500ae4d3d936b1ccdd9106c5e82c56751bcf0625ed36131a51a20a1c0c
 export GRAFANA_VERSION=2.5.0
 export GRAFANA_HOME=$INSTALL/grafana-"$GRAFANA_VERSION"
 export GRAFANA_TARBALL=grafana-"$GRAFANA_VERSION".tar.gz
 export GRAFANA_HASH=d3eaa2c45ae9f8e7424a7b0b74fa8c8360bd25a1f49545d8fb5a874ebf0530fe
 # Configuration for running 'accumulo-proxy' plugin
 export PROXY_REPO=/path/to/accumulo-proxy
 export PROXY_VERSION=2.0.0-SNAPSHOT
 export PROXY_TARBALL=accumulo-proxy-"$PROXY_VERSION"-bin.tar.gz
 export PROXY_HOME=$INSTALL/accumulo-proxy-"$PROXY_VERSION"

 # Performance Profiles
 # --------------------
 PERFORMANCE_PROFILE=8GX2

 case "$PERFORMANCE_PROFILE" in
   8GX2)
     # Enable Accumulo native map
     export ACCUMULO_USE_NATIVE_MAP=false
     # Accumulo tserver jvm heap size
     export ACCUMULO_TSERV_MEM=768m
     # Accumulo data cache size. Only applies when using Accumulo 1.x
     export ACCUMULO_DCACHE_SIZE=256M
     # Accumulo index cache size. Only applies when using Accumulo 1.x
     export ACCUMULO_ICACHE_SIZE=128M
     # Accumulo in mem map size. Only applies when using Accumulo 1.x
     export ACCUMULO_IMAP_SIZE=128M
     # Yarn Node Manager max memory (in MB)
     export YARN_NM_MEM_MB=8192
     # Yarn Node Manager max cpu vcores
     export YARN_NM_CPU_VCORES=8
     # Number of threads per worker process
     export FLUO_WORKER_THREADS=10
     # Worker process memory
     export FLUO_WORKER_MEM_MB=1024
     # Number worker processes
     export FLUO_WORKER_INSTANCES=1
     ;;
   32GX4)
     export ACCUMULO_USE_NATIVE_MAP=false
     export ACCUMULO_TSERV_MEM=6G
     # Accumulo configs below only apply when using Accumulo 1.x
     export ACCUMULO_DCACHE_SIZE=2G
     export ACCUMULO_ICACHE_SIZE=1G
     export ACCUMULO_IMAP_SIZE=1G
     export YARN_NM_MEM_MB=16384
     export YARN_NM_CPU_VCORES=8
     export FLUO_WORKER_THREADS=64
     export FLUO_WORKER_MEM_MB=4096
     export FLUO_WORKER_INSTANCES=2
     ;;
   *)
    echo "Unknown performance profile"
    exit 1;
    ;;
 esac
	#! /usr/bin/env bash
	# From load-env.sh, variable UNO_HOME defaults to root directory of this Uno repo.

	# Versions
	# --------
	export HADOOP_VERSION=${HADOOP_VERSION:-3.3.4}
	export ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.8.0}
	export ACCUMULO_VERSION=${ACCUMULO_VERSION:-2.1.0}
	export FLUO_VERSION=${FLUO_VERSION:-1.2.0}
	export FLUO_YARN_VERSION=${FLUO_YARN_VERSION:-1.0.0}

	# Hashes (supports MD5, SHA-1, SHA-256, and SHA-512; MD5 requires md5sum)
	# --------------
	# Hashes below match default versions above. If you change a version above,
	# you must also change the hash below.
	HADOOP_HASH=$(grep -F hadoop:"${HADOOP_VERSION}": "${UNO_HOME:?}"/conf/checksums \| cut -d : -f 3)
	ZOOKEEPER_HASH=$(grep -F zookeeper:"${ZOOKEEPER_VERSION}": "${UNO_HOME:?}"/conf/checksums \| cut -d : -f 3)
	ACCUMULO_HASH=$(grep -F accumulo:"${ACCUMULO_VERSION}": "${UNO_HOME:?}"/conf/checksums \| cut -d : -f 3)
	FLUO_HASH=037f89cd2bfdaf76a1368256c52de46d6b9a85c9c1bfc776ec4447d02c813fb2
	FLUO_YARN_HASH=c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff874b0
	export HADOOP_HASH ZOOKEEPER_HASH ACCUMULO_HASH FLUO_HASH FLUO_YARN_HASH

	# Network configuration
	# ---------------------
	# Hostname to use in configuration. Set to $(hostname) to use your hostname
	export UNO_HOST=${UNO_HOST:-localhost}

	# Download configuration
	# ----------------------
	# Directory where all depedency packages are downloaded
	export DOWNLOADS=${UNO_DOWNLOADS:-$UNO_HOME/downloads}
	# Tarball file names
	export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
	export HADOOP_TARBALL=hadoop-"$HADOOP_VERSION".tar.gz
	export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
	export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
	export ZOOKEEPER_TARBALL=apache-zookeeper-"$ZOOKEEPER_VERSION"-bin.tar.gz
	if [[ $ZOOKEEPER_VERSION =~ ^3[.][01234].*$ ]]; then
	export ZOOKEEPER_TARBALL=zookeeper-"$ZOOKEEPER_VERSION".tar.gz
	fi
	# Building Accumulo
	#------------------
	# If set, 'uno fetch' will build (instead of downloading) an Accumulo tarball
	# from that directory and copy it to the downloads directory.

	#export ACCUMULO_REPO=/path/to/accumulo

	# Comment out the following if block if you don't want to automatically detect
	# version from the pom.xml. This could be useful if you want to switch branches
	# in your workspace and don't want the detected version to change.
	if [[ -n "$ACCUMULO_REPO" ]]; then
	# Detect the version from the accumulo pom.xml in the workspace
	ACCUMULO_VERSION=$(xmllint --shell "$ACCUMULO_REPO"/pom.xml <<<'xpath /[local-name()="project"]/[local-name()="version"]/text()' \| grep content= \| cut -f2 -d=)
	export ACCUMULO_VERSION
	export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
	fi

	# Building Fluo
	# -------------
	# If set, 'uno fetch' will build (rather than download) a Fluo tarball
	# from that directory and copy it to the downloads directory.

	#export FLUO_REPO=/path/to/fluo

	# Comment out the following if block if you don't want to automatically detect
	# version from the pom.xml. This could be useful if you want to switch branches
	# in your workspace and don't want the detected version to change.
	if [[ -n "$FLUO_REPO" ]]; then
	# Detect the version from the accumulo pom.xml in the workspace
	FLUO_VERSION=$(xmllint --shell "$FLUO_REPO"/pom.xml <<<'xpath /[local-name()="project"]/[local-name()="version"]/text()' \| grep content= \| cut -f2 -d=)
	export FLUO_VERSION
	export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
	fi

	# Building Fluo YARN
	# ------------------
	# If set, 'uno fetch' will build (rather than download) a Fluo YARN tarball
	# from that directory and copy it to the downloads directory.

	#export FLUO_YARN_REPO=/path/to/fluo-yarn

	# Comment out the following if block if you don't want to automatically detect
	# version from the pom.xml. This could be useful if you want to switch branches
	# in your workspace and don't want the detected version to change.
	if [[ -n "$FLUO_YARN_REPO" ]]; then
	# Detect the version from the accumulo pom.xml in the workspace
	FLUO_YARN_VERSION=$(xmllint --shell "$FLUO_YARN_REPO"/pom.xml <<<'xpath /[local-name()="project"]/[local-name()="version"]/text()' \| grep content= \| cut -f2 -d=)
	export FLUO_YARN_VERSION
	export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
	fi

	# Version compatibility checks
	# ----------------------------
	# Apache built Accumulo 1.x only works with Hadoop 2.x. Apache built Accumulo
	# 2.x only works with Hadoop 3.x. However if building Accumulo 1.x, it can
	# work with Hadoop 3.x. The following sanity checks fail fast when versions
	# are incompatible. If the checks cause a problem, comment them out and
	# consider submitting an issue or PR.
	if [[ "${HADOOP_VERSION}" = 1.* ]]; then
	echo "ERROR from uno.conf : Hadoop 1.x is not supported" 1>&2
	exit 1
	fi

	if [[ -z $ACCUMULO_REPO && ${ACCUMULO_VERSION} =~ ^1[.][0-9][.][0-9]+$ && ! ${HADOOP_VERSION} = 2.* ]]; then
	echo "ERROR from uno.conf : When using Accumulo versions prior to 1.10.x, expect Hadoop 2.x not $HADOOP_VERSION" 1>&2
	exit 1
	fi

	if [[ "${ACCUMULO_VERSION}" = 2.* && ! "${HADOOP_VERSION}" = 3.* ]]; then
	echo "ERROR from uno.conf : When using Accumulo 2.x, expect Hadoop 3.x not $HADOOP_VERSION" 1>&2
	exit 1
	fi

	# Installation directories
	# ------------------------
	# Software installation directory
	export INSTALL=$UNO_HOME/install
	# Directory where all Fluo dependencies store their data
	export DATA_DIR=$INSTALL/data
	# Home directories
	export ZOOKEEPER_HOME=$INSTALL/apache-zookeeper-$ZOOKEEPER_VERSION-bin
	if [[ $ZOOKEEPER_VERSION =~ ^3[.][01234].*$ ]]; then
	export ZOOKEEPER_HOME=$INSTALL/zookeeper-$ZOOKEEPER_VERSION
	fi
	export HADOOP_HOME=$INSTALL/hadoop-$HADOOP_VERSION
	export ACCUMULO_HOME=$INSTALL/accumulo-$ACCUMULO_VERSION
	export FLUO_HOME=$INSTALL/fluo-$FLUO_VERSION
	export FLUO_YARN_HOME=$INSTALL/fluo-yarn-$FLUO_YARN_VERSION
	# Config directories
	export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
	# Log directories
	export LOGS_DIR=$INSTALL/logs
	export ACCUMULO_LOG_DIR=$LOGS_DIR/accumulo
	export HADOOP_LOG_DIR=$LOGS_DIR/hadoop
	export ZOO_LOG_DIR=$LOGS_DIR/zookeeper

	# Accumulo configuration
	# ----------------------
	# Accumulo instance name
	export ACCUMULO_INSTANCE=uno
	# Accumulo user
	export ACCUMULO_USER=root
	# Accumulo password
	export ACCUMULO_PASSWORD=secret
	# Number of tservers to run
	export NUM_TSERVERS=1

	# Plugin configuration
	# --------------------
	# Post-install plugins. Example: "influx-metrics accumulo-encryption"
	export POST_INSTALL_PLUGINS=""
	# Post-run plugins. Example: "spark"
	export POST_RUN_PLUGINS=""
	# Configuration for 'spark' plugin
	export SPARK_VERSION=${SPARK_VERSION:-2.3.2}
	export SPARK_HOME=$INSTALL/spark-${SPARK_VERSION}-bin-without-hadoop
	export SPARK_TARBALL=spark-${SPARK_VERSION}-bin-without-hadoop.tgz
	SPARK_HASH=$(grep -F spark:"${SPARK_VERSION}": "${UNO_HOME:?}"/conf/checksums \| cut -d : -f 3)
	export SPARK_HASH
	# Configuration for 'influxdb-metrics' plugin
	# InfluxDB metrics can only be set up on Linux. Mac OS X is not supported.
	export INFLUXDB_VERSION=0.9.4.2
	export INFLUXDB_HOME=$INSTALL/influxdb-"$INFLUXDB_VERSION"
	export INFLUXDB_TARBALL=influxdb-"$INFLUXDB_VERSION".tar.gz
	export INFLUXDB_HASH=fe4269500ae4d3d936b1ccdd9106c5e82c56751bcf0625ed36131a51a20a1c0c
	export GRAFANA_VERSION=2.5.0
	export GRAFANA_HOME=$INSTALL/grafana-"$GRAFANA_VERSION"
	export GRAFANA_TARBALL=grafana-"$GRAFANA_VERSION".tar.gz
	export GRAFANA_HASH=d3eaa2c45ae9f8e7424a7b0b74fa8c8360bd25a1f49545d8fb5a874ebf0530fe
	# Configuration for running 'accumulo-proxy' plugin
	export PROXY_REPO=/path/to/accumulo-proxy
	export PROXY_VERSION=2.0.0-SNAPSHOT
	export PROXY_TARBALL=accumulo-proxy-"$PROXY_VERSION"-bin.tar.gz
	export PROXY_HOME=$INSTALL/accumulo-proxy-"$PROXY_VERSION"

	# Performance Profiles
	# --------------------
	PERFORMANCE_PROFILE=8GX2

	case "$PERFORMANCE_PROFILE" in
	8GX2)
	# Enable Accumulo native map
	export ACCUMULO_USE_NATIVE_MAP=false
	# Accumulo tserver jvm heap size
	export ACCUMULO_TSERV_MEM=768m
	# Accumulo data cache size. Only applies when using Accumulo 1.x
	export ACCUMULO_DCACHE_SIZE=256M
	# Accumulo index cache size. Only applies when using Accumulo 1.x
	export ACCUMULO_ICACHE_SIZE=128M
	# Accumulo in mem map size. Only applies when using Accumulo 1.x
	export ACCUMULO_IMAP_SIZE=128M
	# Yarn Node Manager max memory (in MB)
	export YARN_NM_MEM_MB=8192
	# Yarn Node Manager max cpu vcores
	export YARN_NM_CPU_VCORES=8
	# Number of threads per worker process
	export FLUO_WORKER_THREADS=10
	# Worker process memory
	export FLUO_WORKER_MEM_MB=1024
	# Number worker processes
	export FLUO_WORKER_INSTANCES=1
	;;
	32GX4)
	export ACCUMULO_USE_NATIVE_MAP=false
	export ACCUMULO_TSERV_MEM=6G
	# Accumulo configs below only apply when using Accumulo 1.x
	export ACCUMULO_DCACHE_SIZE=2G
	export ACCUMULO_ICACHE_SIZE=1G
	export ACCUMULO_IMAP_SIZE=1G
	export YARN_NM_MEM_MB=16384
	export YARN_NM_CPU_VCORES=8
	export FLUO_WORKER_THREADS=64
	export FLUO_WORKER_MEM_MB=4096
	export FLUO_WORKER_INSTANCES=2
	;;
	*)
	echo "Unknown performance profile"
	exit 1;
	;;
	esac