conf/uno.conf - fluo-uno - Git at Google

 #! /usr/bin/env bash
 # From load-env.sh, variable UNO_HOME defaults to root directory of this Uno repo.

 # Versions
 # --------
 export HADOOP_VERSION=${HADOOP_VERSION:-2.8.4}
 export ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.4.12}
 export ACCUMULO_VERSION=${ACCUMULO_VERSION:-1.9.2}
 export SPARK_VERSION=${SPARK_VERSION:-1.6.3}
 export FLUO_VERSION=${FLUO_VERSION:-1.2.0}
 export FLUO_YARN_VERSION=${FLUO_YARN_VERSION:-1.0.0}

 # Hashes (supports MD5, SHA-1, SHA-256, and SHA-512; MD5 requires md5sum)
 # --------------
 # Hashes below match default versions above. If you change a version above,
 # you must also change the hash below.
 export FLUO_HASH=037f89cd2bfdaf76a1368256c52de46d6b9a85c9c1bfc776ec4447d02c813fb2
 export FLUO_YARN_HASH=c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff874b0
 export HADOOP_HASH=$(grep -F hadoop:${HADOOP_VERSION}: $UNO_HOME/conf/checksums | cut -d : -f 3)
 export ZOOKEEPER_HASH=$(grep -F zookeeper:${ZOOKEEPER_VERSION}: $UNO_HOME/conf/checksums | cut -d : -f 3)
 export ACCUMULO_HASH=$(grep -F accumulo:${ACCUMULO_VERSION}: $UNO_HOME/conf/checksums | cut -d : -f 3)
 export SPARK_HASH=d13358a2d45e78d7c8cf22656d63e5715a5900fab33b3340df9e11ce3747e314
 export INFLUXDB_HASH=fe4269500ae4d3d936b1ccdd9106c5e82c56751bcf0625ed36131a51a20a1c0c
 export GRAFANA_HASH=d3eaa2c45ae9f8e7424a7b0b74fa8c8360bd25a1f49545d8fb5a874ebf0530fe

 # Network configuration
 # ---------------------
 # Hostname to use in configuration. Set to $(hostname) to use your hostname
 export UNO_HOST=${UNO_HOST:-localhost}

 # Download configuration
 # ----------------------
 # Directory where all depedency packages are downloaded
 export DOWNLOADS=$UNO_HOME/downloads
 # Tarball file names
 export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
 export HADOOP_TARBALL=hadoop-"$HADOOP_VERSION".tar.gz
 export ZOOKEEPER_TARBALL=zookeeper-"$ZOOKEEPER_VERSION".tar.gz
 export SPARK_TARBALL=spark-$SPARK_VERSION-bin-without-hadoop.tgz
 export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
 export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz

 # Building Accumulo
 #------------------
 # If set, 'uno fetch' will build (instead of downloading) an Accumulo tarball
 # from that directory and copy it to the downloads directory.

 #export ACCUMULO_REPO=/path/to/accumulo

 # Comment out the following if block if you don't want to automatically detect
 # version from the pom.xml. This could be useful if you want to switch branches
 # in your workspace and don't want the detected version to change.
 if [[ -n "$ACCUMULO_REPO" ]]; then
   # Detect the version from the accumulo pom.xml in the workspace
   ACCUMULO_VERSION=$(xmllint --shell "$ACCUMULO_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
   export ACCUMULO_VERSION
   export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
 fi

 # Building Fluo
 # -------------
 # If set, 'uno fetch' will build (rather than download) a Fluo tarball
 # from that directory and copy it to the downloads directory.

 #export FLUO_REPO=/path/to/fluo

 # Comment out the following if block if you don't want to automatically detect
 # version from the pom.xml. This could be useful if you want to switch branches
 # in your workspace and don't want the detected version to change.
 if [[ -n "$FLUO_REPO" ]]; then
   # Detect the version from the accumulo pom.xml in the workspace
   FLUO_VERSION=$(xmllint --shell "$FLUO_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
   export FLUO_VERSION
   export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
 fi

 # Building Fluo YARN
 # ------------------
 # If set, 'uno fetch' will build (rather than download) a Fluo YARN tarball
 # from that directory and copy it to the downloads directory.

 #export FLUO_YARN_REPO=/path/to/fluo-yarn

 # Comment out the following if block if you don't want to automatically detect
 # version from the pom.xml. This could be useful if you want to switch branches
 # in your workspace and don't want the detected version to change.
 if [[ -n "$FLUO_YARN_REPO" ]]; then
   # Detect the version from the accumulo pom.xml in the workspace
   FLUO_YARN_VERSION=$(xmllint --shell "$FLUO_YARN_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
   export FLUO_YARN_VERSION
   export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
 fi

 # Installation directories
 # ------------------------
 # Software installation directory
 export INSTALL=$UNO_HOME/install
 # Directory where all Fluo dependencies store their data
 export DATA_DIR=$INSTALL/data
 # Home directories
 export ZOOKEEPER_HOME=$INSTALL/zookeeper-$ZOOKEEPER_VERSION
 export HADOOP_HOME=$INSTALL/hadoop-$HADOOP_VERSION
 if [[ $HADOOP_VERSION =~ ^2\..*$ ]]; then
   export HADOOP_PREFIX=$HADOOP_HOME
 fi
 export ACCUMULO_HOME=$INSTALL/accumulo-$ACCUMULO_VERSION
 export SPARK_HOME=$INSTALL/spark-$SPARK_VERSION-bin-without-hadoop
 export FLUO_HOME=$INSTALL/fluo-$FLUO_VERSION
 export FLUO_YARN_HOME=$INSTALL/fluo-yarn-$FLUO_YARN_VERSION
 # Config directories
 export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
 # Log directories
 export LOGS_DIR=$INSTALL/logs
 export ACCUMULO_LOG_DIR=$LOGS_DIR/accumulo
 export HADOOP_LOG_DIR=$LOGS_DIR/hadoop
 export ZOO_LOG_DIR=$LOGS_DIR/zookeeper

 # Accumulo configuration
 # ----------------------
 # Accumulo instance name
 export ACCUMULO_INSTANCE=uno
 # Accumulo user
 export ACCUMULO_USER=root
 # Accumulo password
 export ACCUMULO_PASSWORD=secret

 # Metrics configuration
 # ---------------------
 # Metrics can only be set up on Linux. Mac OS X is not supported.
 export INFLUXDB_VERSION=0.9.4.2
 export INFLUXDB_HOME=$INSTALL/influxdb-"$INFLUXDB_VERSION"
 export GRAFANA_VERSION=2.5.0
 export GRAFANA_HOME=$INSTALL/grafana-"$GRAFANA_VERSION"

 #Performance Profiles
 #--------------------

 PERFORMACE_PROFILE=8GX2

 case "$PERFORMACE_PROFILE" in
   8GX2)
     # Enable Accumulo native map
     export ACCUMULO_USE_NATIVE_MAP=false
     # Accumulo tserver jvm heap size
     export ACCUMULO_TSERV_MEM=768m
     # Accumulo data cache size. Only applies when using Accumulo 1.x
     export ACCUMULO_DCACHE_SIZE=256M
     # Accumulo index cache size. Only applies when using Accumulo 1.x
     export ACCUMULO_ICACHE_SIZE=128M
     # Accumulo in mem map size. Only applies when using Accumulo 1.x
     export ACCUMULO_IMAP_SIZE=128M
     # Yarn Node Manager max memory (in MB)
     export YARN_NM_MEM_MB=8192
     # Yarn Node Manager max cpu vcores
     export YARN_NM_CPU_VCORES=8
     # Number of threads per worker process
     export FLUO_WORKER_THREADS=10
     # Worker process memory
     export FLUO_WORKER_MEM_MB=1024
     # Number worker processes
     export FLUO_WORKER_INSTANCES=1
     ;;
   32GX4)
     export ACCUMULO_USE_NATIVE_MAP=false
     export ACCUMULO_TSERV_MEM=6G
     # Accumulo configs below only apply when using Accumulo 1.x
     export ACCUMULO_DCACHE_SIZE=2G
     export ACCUMULO_ICACHE_SIZE=1G
     export ACCUMULO_IMAP_SIZE=1G
     export YARN_NM_MEM_MB=16384
     export YARN_NM_CPU_VCORES=8
     export FLUO_WORKER_THREADS=64
     export FLUO_WORKER_MEM_MB=4096
     export FLUO_WORKER_INSTANCES=2
     ;;
   *)
    echo "Unknown performance profile"
    exit 1;
    ;;
 esac
	#! /usr/bin/env bash
	# From load-env.sh, variable UNO_HOME defaults to root directory of this Uno repo.

	# Versions
	# --------
	export HADOOP_VERSION=${HADOOP_VERSION:-2.8.4}
	export ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.4.12}
	export ACCUMULO_VERSION=${ACCUMULO_VERSION:-1.9.2}
	export SPARK_VERSION=${SPARK_VERSION:-1.6.3}
	export FLUO_VERSION=${FLUO_VERSION:-1.2.0}
	export FLUO_YARN_VERSION=${FLUO_YARN_VERSION:-1.0.0}

	# Hashes (supports MD5, SHA-1, SHA-256, and SHA-512; MD5 requires md5sum)
	# --------------
	# Hashes below match default versions above. If you change a version above,
	# you must also change the hash below.
	export FLUO_HASH=037f89cd2bfdaf76a1368256c52de46d6b9a85c9c1bfc776ec4447d02c813fb2
	export FLUO_YARN_HASH=c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff874b0
	export HADOOP_HASH=$(grep -F hadoop:${HADOOP_VERSION}: $UNO_HOME/conf/checksums \| cut -d : -f 3)
	export ZOOKEEPER_HASH=$(grep -F zookeeper:${ZOOKEEPER_VERSION}: $UNO_HOME/conf/checksums \| cut -d : -f 3)
	export ACCUMULO_HASH=$(grep -F accumulo:${ACCUMULO_VERSION}: $UNO_HOME/conf/checksums \| cut -d : -f 3)
	export SPARK_HASH=d13358a2d45e78d7c8cf22656d63e5715a5900fab33b3340df9e11ce3747e314
	export INFLUXDB_HASH=fe4269500ae4d3d936b1ccdd9106c5e82c56751bcf0625ed36131a51a20a1c0c
	export GRAFANA_HASH=d3eaa2c45ae9f8e7424a7b0b74fa8c8360bd25a1f49545d8fb5a874ebf0530fe

	# Network configuration
	# ---------------------
	# Hostname to use in configuration. Set to $(hostname) to use your hostname
	export UNO_HOST=${UNO_HOST:-localhost}

	# Download configuration
	# ----------------------
	# Directory where all depedency packages are downloaded
	export DOWNLOADS=$UNO_HOME/downloads
	# Tarball file names
	export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
	export HADOOP_TARBALL=hadoop-"$HADOOP_VERSION".tar.gz
	export ZOOKEEPER_TARBALL=zookeeper-"$ZOOKEEPER_VERSION".tar.gz
	export SPARK_TARBALL=spark-$SPARK_VERSION-bin-without-hadoop.tgz
	export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
	export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz

	# Building Accumulo
	#------------------
	# If set, 'uno fetch' will build (instead of downloading) an Accumulo tarball
	# from that directory and copy it to the downloads directory.

	#export ACCUMULO_REPO=/path/to/accumulo

	# Comment out the following if block if you don't want to automatically detect
	# version from the pom.xml. This could be useful if you want to switch branches
	# in your workspace and don't want the detected version to change.
	if [[ -n "$ACCUMULO_REPO" ]]; then
	# Detect the version from the accumulo pom.xml in the workspace
	ACCUMULO_VERSION=$(xmllint --shell "$ACCUMULO_REPO"/pom.xml <<<'xpath /[local-name()="project"]/[local-name()="version"]/text()' \| grep content= \| cut -f2 -d=)
	export ACCUMULO_VERSION
	export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
	fi

	# Building Fluo
	# -------------
	# If set, 'uno fetch' will build (rather than download) a Fluo tarball
	# from that directory and copy it to the downloads directory.

	#export FLUO_REPO=/path/to/fluo

	# Comment out the following if block if you don't want to automatically detect
	# version from the pom.xml. This could be useful if you want to switch branches
	# in your workspace and don't want the detected version to change.
	if [[ -n "$FLUO_REPO" ]]; then
	# Detect the version from the accumulo pom.xml in the workspace
	FLUO_VERSION=$(xmllint --shell "$FLUO_REPO"/pom.xml <<<'xpath /[local-name()="project"]/[local-name()="version"]/text()' \| grep content= \| cut -f2 -d=)
	export FLUO_VERSION
	export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
	fi

	# Building Fluo YARN
	# ------------------
	# If set, 'uno fetch' will build (rather than download) a Fluo YARN tarball
	# from that directory and copy it to the downloads directory.

	#export FLUO_YARN_REPO=/path/to/fluo-yarn

	# Comment out the following if block if you don't want to automatically detect
	# version from the pom.xml. This could be useful if you want to switch branches
	# in your workspace and don't want the detected version to change.
	if [[ -n "$FLUO_YARN_REPO" ]]; then
	# Detect the version from the accumulo pom.xml in the workspace
	FLUO_YARN_VERSION=$(xmllint --shell "$FLUO_YARN_REPO"/pom.xml <<<'xpath /[local-name()="project"]/[local-name()="version"]/text()' \| grep content= \| cut -f2 -d=)
	export FLUO_YARN_VERSION
	export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
	fi

	# Installation directories
	# ------------------------
	# Software installation directory
	export INSTALL=$UNO_HOME/install
	# Directory where all Fluo dependencies store their data
	export DATA_DIR=$INSTALL/data
	# Home directories
	export ZOOKEEPER_HOME=$INSTALL/zookeeper-$ZOOKEEPER_VERSION
	export HADOOP_HOME=$INSTALL/hadoop-$HADOOP_VERSION
	if [[ $HADOOP_VERSION =~ ^2\..*$ ]]; then
	export HADOOP_PREFIX=$HADOOP_HOME
	fi
	export ACCUMULO_HOME=$INSTALL/accumulo-$ACCUMULO_VERSION
	export SPARK_HOME=$INSTALL/spark-$SPARK_VERSION-bin-without-hadoop
	export FLUO_HOME=$INSTALL/fluo-$FLUO_VERSION
	export FLUO_YARN_HOME=$INSTALL/fluo-yarn-$FLUO_YARN_VERSION
	# Config directories
	export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
	# Log directories
	export LOGS_DIR=$INSTALL/logs
	export ACCUMULO_LOG_DIR=$LOGS_DIR/accumulo
	export HADOOP_LOG_DIR=$LOGS_DIR/hadoop
	export ZOO_LOG_DIR=$LOGS_DIR/zookeeper

	# Accumulo configuration
	# ----------------------
	# Accumulo instance name
	export ACCUMULO_INSTANCE=uno
	# Accumulo user
	export ACCUMULO_USER=root
	# Accumulo password
	export ACCUMULO_PASSWORD=secret

	# Metrics configuration
	# ---------------------
	# Metrics can only be set up on Linux. Mac OS X is not supported.
	export INFLUXDB_VERSION=0.9.4.2
	export INFLUXDB_HOME=$INSTALL/influxdb-"$INFLUXDB_VERSION"
	export GRAFANA_VERSION=2.5.0
	export GRAFANA_HOME=$INSTALL/grafana-"$GRAFANA_VERSION"

	#Performance Profiles
	#--------------------

	PERFORMACE_PROFILE=8GX2

	case "$PERFORMACE_PROFILE" in
	8GX2)
	# Enable Accumulo native map
	export ACCUMULO_USE_NATIVE_MAP=false
	# Accumulo tserver jvm heap size
	export ACCUMULO_TSERV_MEM=768m
	# Accumulo data cache size. Only applies when using Accumulo 1.x
	export ACCUMULO_DCACHE_SIZE=256M
	# Accumulo index cache size. Only applies when using Accumulo 1.x
	export ACCUMULO_ICACHE_SIZE=128M
	# Accumulo in mem map size. Only applies when using Accumulo 1.x
	export ACCUMULO_IMAP_SIZE=128M
	# Yarn Node Manager max memory (in MB)
	export YARN_NM_MEM_MB=8192
	# Yarn Node Manager max cpu vcores
	export YARN_NM_CPU_VCORES=8
	# Number of threads per worker process
	export FLUO_WORKER_THREADS=10
	# Worker process memory
	export FLUO_WORKER_MEM_MB=1024
	# Number worker processes
	export FLUO_WORKER_INSTANCES=1
	;;
	32GX4)
	export ACCUMULO_USE_NATIVE_MAP=false
	export ACCUMULO_TSERV_MEM=6G
	# Accumulo configs below only apply when using Accumulo 1.x
	export ACCUMULO_DCACHE_SIZE=2G
	export ACCUMULO_ICACHE_SIZE=1G
	export ACCUMULO_IMAP_SIZE=1G
	export YARN_NM_MEM_MB=16384
	export YARN_NM_CPU_VCORES=8
	export FLUO_WORKER_THREADS=64
	export FLUO_WORKER_MEM_MB=4096
	export FLUO_WORKER_INSTANCES=2
	;;
	*)
	echo "Unknown performance profile"
	exit 1;
	;;
	esac