| #!/usr/bin/env bash |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| |
| ## Environment configuration |
| # Hadoop installation |
| # set var only if it has not already been set externally |
| if [ -z "${HADOOP_PREFIX}" ] ; then |
| export HADOOP_PREFIX= |
| fi |
| # Base directory for gridmix install |
| # set var only if it has not already been set externally |
| if [ -z "${GRID_MIX_HOME}" ] ; then |
| export GRID_MIX_HOME=${GRID_DIR} |
| fi |
| # Hadoop example jar |
| # set var only if it has not already been set externally |
| if [ -z "${EXAMPLE_JAR}" ] ; then |
| export EXAMPLE_JAR="${HADOOP_PREFIX}/hadoop-*examples.jar" |
| fi |
| # Hadoop test jar |
| # set var only if it has not already been set externally |
| if [ -z "${APP_JAR}" ] ; then |
| export APP_JAR="${HADOOP_PREFIX}/hadoop-*test.jar" |
| fi |
| # Hadoop streaming jar |
| # set var only if it has not already been set externally |
| if [ -z "${STREAM_JAR}" ] ; then |
| export STREAM_JAR="${HADOOP_PREFIX}/contrib/streaming/hadoop-*streaming.jar" |
| fi |
| # Location on default filesystem for writing gridmix data (usually HDFS) |
| # Default: /gridmix/data |
| # set var only if it has not already been set externally |
| if [ -z "${GRID_MIX_DATA}" ] ; then |
| export GRID_MIX_DATA=/gridmix/data |
| fi |
| # Location of executables in default filesystem (usually HDFS) |
| # Default: /gridmix/programs |
| # set var only if it has not already been set externally |
| if [ -z "${GRID_MIX_PROG}" ] ; then |
| export GRID_MIX_PROG=/gridmix/programs |
| fi |
| |
| ## Data sources |
| # Variable length key, value compressed SequenceFile |
| export VARCOMPSEQ=${GRID_MIX_DATA}/WebSimulationBlockCompressed |
| # Fixed length key, value compressed SequenceFile |
| export FIXCOMPSEQ=${GRID_MIX_DATA}/MonsterQueryBlockCompressed |
| # Variable length key, value uncompressed Text File |
| export VARINFLTEXT=${GRID_MIX_DATA}/SortUncompressed |
| # Fixed length key, value compressed Text File |
| export FIXCOMPTEXT=${GRID_MIX_DATA}/EntropySimulationCompressed |
| |
| ## Job sizing |
| export NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS=5 |
| export NUM_OF_LARGE_JOBS_PER_CLASS=3 |
| export NUM_OF_MEDIUM_JOBS_PER_CLASS=20 |
| export NUM_OF_SMALL_JOBS_PER_CLASS=40 |
| |
| export NUM_OF_REDUCERS_FOR_LARGE_JOB=370 |
| export NUM_OF_REDUCERS_FOR_MEDIUM_JOB=170 |
| export NUM_OF_REDUCERS_FOR_SMALL_JOB=15 |
| |
| ## Throttling |
| export INTERVAL_BETWEEN_SUBMITION=20 |
| |
| ## Hod |
| #export HOD_OPTIONS="" |
| |
| export CLUSTER_DIR_BASE=$GRID_MIX_HOME/CLUSTER_DIR_BASE |
| export HOD_CONFIG= |
| export ALL_HOD_OPTIONS="-c ${HOD_CONFIG} ${HOD_OPTIONS}" |
| export SMALL_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 5" |
| export MEDIUM_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 50" |
| export LARGE_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 100" |
| |