installer/trafodion_install

#!/bin/bash

# @@@ START COPYRIGHT @@@
#
# (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
# @@@ END COPYRIGHT @@@


#==============================================
#  Trafodion installation script. Setup Trafodion
#  environment, configuration of HBase with
#  Hbase-trx and co-processors need for Trafodion, and
#  install Trafodion build. All parameters from scripts
#  use configuration file.
#==============================================

function print_usage {
cat << EOF

This script will install Trafodion. It will create a configuration
file (if one has not been created), setup of the environment needed
for Trafodion, configure HBase with Hbase-trx and co-processors needed,
and install a specified Trafodion build.

Options:
    --help             Print this message and exit
    --accept_license   If provided, the user agrees to accept all the
                       provisions in the Trafodion license.  This allows
                       for automation by skipping the display and prompt of
                       the Trafodion license.
    --config_file      If provided, all install prompts will be
                       taken from this file and not prompted for.
    --run_scanner      If provided, the Trafodion scanner will run
                       to check system configuration.
    --no_scanner       If provided, the Trafodion scanner will not
                       run to check system configuration.
EOF
}

#==============================================
#Parse input
USER_CONFIG=""
ACCEPT_LICENSE="N"
RUN_SCANNER="Y"

while [[ $# -gt 0 ]]; do
    case "$1" in
        --help)
            print_usage
            exit -1
            ;;
        --accept_license)
            ACCEPT_LICENSE="Y"
            ;;
        --config_file)
            if [[ -z "$2" ]]; then
                echo "***ERROR: No value passed to param $1."
                print_usage
                exit -1
            fi
            USER_CONFIG=$2
            if [ -f $USER_CONFIG ]; then
               source $USER_CONFIG
            else
               echo "***ERROR: configuration file $USER_CONFIG does not exist"
               exit -1
            fi
            shift
            ;;
        --run_scanner)
            RUN_SCANNER="Y"
            selectedRun="true"
            if [[ $selectedNo == "true" ]]; then
               echo "***ERROR: Selected --run_scanner and --no_scanner. Invalid combination."
               exit -1
            fi
            ;;
        --no_scanner)
            RUN_SCANNER="N"
            selectedNo="true"
            if [[ $selectedRun == "true" ]]; then
               echo "***ERROR: Selected --run_scanner and --no_scanner. Invalid combination."
               exit -1 
            fi
            ;;
        *)
            echo "***ERROR: unknown parameter '$1'"
            print_usage
            exit -1
    esac
    shift
done


#==============================================
echo
echo "******************************"
echo " TRAFODION INSTALLATION START"
echo "******************************"
echo

#==============================================
# Default values

export timestamp=$(date +%F-%H-%M-%S)
export TRAF_LOGDIR="/var/log/trafodion"
export INSTALL_LOG="$TRAF_LOGDIR/trafodion_install_$timestamp.log"
export SCANNER_LOG="$TRAF_LOGDIR/trafodion_scanner_$timestamp.log"
#export TRAF_WORKDIR="$( cd "$( dirname "$0" )" && pwd )/.."
export TRAF_WORKDIR="/usr/lib/trafodion"
export LOCAL_WORKDIR="$( cd "$( dirname "$0" )" && pwd )"
export TRAF_CONFIG="/etc/trafodion/trafodion_config"
export TRAF_CONFIG_DIR=$(dirname $TRAF_CONFIG)
export TRAF_CONFIG_FILE=$(basename $TRAF_CONFIG)
export TRAF_USER="trafodion"

#==============================================
#Prechecks 

#Check not installing as userid Trafodion
if [[ "$(whoami)" == "$TRAF_USER" ]]; then
   echo "***ERROR: The Trafodion username cannot be the same as the install username"
   exit -1
fi

# check that we are running from a userid with sudo access
sudo echo "***INFO: testing sudo access"
if [ $? -ne 0 ]; then
    echo "***ERROR: This script must be run from a userid with sudo access."
    exit -1
fi

#==============================================
# Setup log & working directories

sudo mkdir -p $TRAF_LOGDIR
sudo chmod 777 $TRAF_LOGDIR
echo "***INFO: Log file located at $INSTALL_LOG"

sudo mkdir -p $TRAF_CONFIG_DIR
sudo chmod 777 $TRAF_CONFIG_DIR
echo "***INFO: Config directory: $TRAF_CONFIG_DIR"

# create a working directory
sudo mkdir -p $TRAF_WORKDIR
sudo chmod -R 777 $TRAF_WORKDIR
echo "***INFO: Working directory: $TRAF_WORKDIR"

# copy install scripts into working directory
sudo cp -r $(cd "$( dirname "$0" )" && pwd)/ $TRAF_WORKDIR
sudo cp -rf $LOCAL_WORKDIR $TRAF_WORKDIR
sudo chmod -R 777 $TRAF_WORKDIR
#=============================================
# Shutdown Trafodion if it is running

userForTrafodion=`grep "$TRAF_USER:" /etc/passwd | wc -l`

if [[ "$userForTrafodion" != "0" ]]; then
   isTrafUp=$(sudo su $TRAF_USER --login --command "sqcheck" 2>/dev/null)

   trafodionRunning=$(echo $isTrafUp | grep "The SQ environment is up" | wc -l)
   if [[ "$trafodionRunning" -eq "1" ]]; then
      echo "***ERROR: Trafodion is still running."
      echo -n "Stop Trafodion (Y/N), default is N: "
      read answer

      if [ -z $answer ]; then
         echo "***ERROR: Please stop Trafodion then rerun trafodion_install"
         echo "***ERROR: Change to trafodion userid then do a 'sqstop'"
         exit -1
      else
         if [[ ${answer^} == "Y" ]]; then
            echo "***INFO: Stopping Trafodion instance."
            sudo su $TRAF_USER --login --command "sqstop"
            sudo su $TRAF_USER --login --command "ckillall"
         else
            echo "***ERROR: Please stop Trafodion then rerun trafodion_install"
            echo "***ERROR: Change to trafodion user id then do a 'sqstop'"
            exit -1
         fi
      fi
   fi
fi

#==============================================
# config file

# if user did not supply their own config file, use the default
# or prompt to generate a new one.
if [ -z "$USER_CONFIG" ]; then
   if [ -f $TRAF_CONFIG ]; then
      # strip out LOCAL_WORKDIR from config file, if there.
      # LOCAL_WORKDIR was erroneously saved to the config file in
      # previous versions of the installer scripts and since the config
      # file hangs around between installer runs it needs to be stripped
      # out here before we source in the config file
      cp $TRAF_CONFIG config_temp
      sudo grep -vE "LOCAL_WORKDIR" config_temp > $TRAF_CONFIG 2> /dev/null
      source $TRAF_CONFIG
   fi

   # prompt user for configuration info
   $LOCAL_WORKDIR/traf_config_setup
   if [ ${PIPESTATUS[0]} != "0" ]; then
      echo "***ERROR: Error while running traf_config_setup." | tee -a $INSTALL_LOG
      echo "***ERROR: No configuration file created." | tee -a $INSTALL_LOG
      exit -1
   fi
else
   # copy the user's config file to the default location
   sudo cp $USER_CONFIG $TRAF_CONFIG
   if [ $? -ne 0 ]; then
      echo "***ERROR: unable to copy $USER_CONFIG to $TRAF_CONFIG"
      exit -1
   fi
   sudo chmod 777 $TRAF_CONFIG

   #Check if install_features file exists
   install_features_path=$(tar -tf $TRAF_BUILD | grep "install_features")

   if [[ ! -z $install_features_path ]]; then
      tar -xzf $TRAF_BUILD $install_features_path
      source $install_features_path
      echo "export CDH_5_3_HDP_2_2_SUPPORT=\"$CDH_5_3_HDP_2_2_SUPPORT\"" >> $TRAF_CONFIG
   else
      CDH_5_3_HDP_2_2_SUPPORT="N"
      echo "export CDH_5_3_HDP_2_2_SUPPORT=\"$CDH_5_3_HDP_2_2_SUPPORT\"" >> $TRAF_CONFIG
   fi

fi

source $TRAF_CONFIG
if [ $? -ne 0 ]; then
   echo "***ERROR: unable to source $TRAF_CONFIG"
   exit -1
fi


#==============================================
#Run Trafodion scanner

if [[ "$RUN_SCANNER" == "Y" ]]; then
   echo "***INFO: Running Trafodion Scanner"
   $LOCAL_WORKDIR/tools/trafodion_scanner -l $SCANNER_LOG -t $TRAF_CONFIG \
   -c $LOCAL_WORKDIR/tools/trafodion_scanner.cfg | tee -a $INSTALL_LOG

   returnCode=${PIPESTATUS[0]}

   case $returnCode in

      2|1)
      echo "***ERROR: Error while running trafodion_scanner."
      echo "***ERROR: Please review logs and fix listed issue(s)."
      exit -1
      ;;
      3|0)
      echo "***INFO: Trafodion scanner ran without error. Install will continue..."
      ;;
      *)
      echo "***ERROR: Error while running trafodion_scanner."
      exit -1
      ;;
   esac
else
   echo "***INFO: Trafodion scanner will not be run."
fi

#=============================================

if [ $node_count -eq 1 ]; then
    TRAF_PDSH=""
else
    TRAF_PDSH="pdsh -S $MY_NODES"
fi

sed -i '/ACCEPT_LICENSE\=/d' $TRAF_CONFIG
echo "export ACCEPT_LICENSE=\"$ACCEPT_LICENSE\"" >> $TRAF_CONFIG

#==============================================
#Check that Hadoop nodes match Trafodion nodes

curlRC=0
if [[ $HADOOP_TYPE == "cloudera" ]]; then
   curl -su $ADMIN:$PASSWORD http://$URL/api/v6/cm/deployment > tempFile
   curlRC=$?
   numberHadoopNodes=$(grep -r "hostname" tempFile | wc -l)
   grep -r "hostname" tempFile > tempFile2
fi

if [[ $HADOOP_TYPE == "hortonworks" ]]; then
   curl -su $ADMIN:$PASSWORD http://$URL/api/v1/clusters/$CLUSTER_NAME/hosts > tempFile
   curlRC=$?
   numberHadoopNodes=$(grep -r "host_name" tempFile | wc -l)
   grep -r "host_name" tempFile > tempFile2
fi

if [ $curlRC != 0 ]; then
   echo "***ERROR: Unable to get list of hosts from $HADOOP_TYPE (RC=$curlRC)"
   echo "***ERROR: curl command failed."
   exit -1
fi

# in most cases curl does not return an error
# so curl's actual output needs to be checked, too
curl_error=$(grep TITLE tempFile | grep Error | wc -l)

if [ $curl_error -ne 0 ]; then
   echo "***ERROR: Unable to get list of hosts from $HADOOP_TYPE"
   cat tempFile
   exit -1
fi

if [[ $numberHadoopNodes != $node_count ]]; then
   echo "***ERROR: Number of Trafodion nodes ($node_count) must match number of $HADOOP_TYPE nodes ($numberHadoopNodes)"
   cat tempFile
   echo "***ERROR: Exiting..."
   rm tempFile
   rm tempFile2
   exit -1
fi

#Get list of all hadoop nodes
hadoopNodes=""
while read line
do
   hostName=$(echo $line | awk '{print $3}' | sed 's/\"//g' | sed 's/\..*//' | sed 's/\,//g')
   hadoopNodes="$hadoopNodes $hostName"
done < tempFile2
rm tempFile
rm tempFile2

#Check to see that each Hadoop node is in the Trafodion list of nodes
for node in $hadoopNodes
do
   if [[ $node_count -eq "1" ]]; then
      #In the case that someone uses "localhost"
      if [[ $(hostname -s) != *$node* ]]; then
         echo "***ERROR: Trafodion must be installed on all $HADOOP_TYPE nodes"
         echo "***ERROR: $HADOOP_TYPE Nodes: $hadoopNodes"
         echo "***ERROR: Trafodion Nodes: $NODE_LIST"
         echo "***ERROR: Please rerun trafodion_install with correct set of nodes"
         echo "***ERROR: Exiting..."
         exit -1
      fi
   else
      if [[ $NODE_LIST != *$node* ]]; then
         echo "***ERROR: Trafodion must be installed on all $HADOOP_TYPE nodes"
         echo "***ERROR: $HADOOP_TYPE Nodes: $hadoopNodes"
         echo "***ERROR: Trafodion Nodes: $NODE_LIST"
         echo "***ERROR: Please rerun trafodion_install with correct set of nodes"
         echo "***ERROR: Exiting..."
         exit -1
      fi
   fi
done

#=============================================
# untar package and files into /usr/lib/trafodion (the new TRAF_WORKDIR)
# This will be a global location that all tar files are untarred into

# if we have the package tar file we need to extract
# trafodion core and dcs  and rest-server builds from it
if [ ! -z $TRAF_PACKAGE ]; then

   # create directory to contain build tar files using the
   # timestamp from the package filename
   traf_filename=$(basename "$TRAF_PACKAGE")
   UNTAR_DIR="$TRAF_WORKDIR/${traf_filename%.tar.gz}"
   sudo mkdir -p $UNTAR_DIR
   sudo chmod 755 $UNTAR_DIR

   # untar the package
   sudo tar -xzf $TRAF_PACKAGE --directory=$UNTAR_DIR

   build_file=$(ls $UNTAR_DIR | grep "trafodion_server")
   if [[ -z $build_file ]]; then
      echo "***ERROR: trafodion_server-*.tgz could not be found in $UNTAR_DIR"
      exit -1
   fi
   TRAF_BUILD=$UNTAR_DIR/$build_file

   build_file=$(ls $UNTAR_DIR | grep "dcs")
   DCS_BUILD=$UNTAR_DIR/$build_file

   build_file=$(ls $UNTAR_DIR | grep "rest")
   if [ -z $build_file ]; then
      REST_BUILD=""
   else
      REST_BUILD=$UNTAR_DIR/$build_file
   fi

else
   # we don't have a package and only have the trafodion and dcs build tar files
   # we can't use the package filename timestamp since we don't have a package tar
   # file so we'll just create a directory using the current time
   UNTAR_DIR="$TRAF_WORKDIR/traf_$(date +%F_%H%M)"
   sudo mkdir -p $UNTAR_DIR
   sudo chmod 777 $UNTAR_DIR

   sudo cp $TRAF_BUILD $UNTAR_DIR
   if [ $? -ne 0 ]; then
      echo "***ERROR: unable to copy trafodion build ($TRAF_BUILD) to $UNTAR_DIR"
      exit -1
   fi

   sudo cp $DCS_BUILD $UNTAR_DIR
   if [ $? -ne 0 ]; then
      echo "***ERROR: unable to copy DCS build ($DCS_BUILD) to $UNTAR_DIR"
      exit -1
   fi

   if [ ! -z $REST_BUILD ]; then
      sudo cp $REST_BUILD $UNTAR_DIR
      if [ $? -ne 0 ]; then
         echo "***ERROR: unable to copy REST build ($REST_BUILD) to $UNTAR_DIR"
         exit -1
      fi
   fi

   build_file=$(basename $TRAF_BUILD)
   TRAF_BUILD=$UNTAR_DIR/$build_file

   sudo tar -xzf $TRAF_BUILD --directory=$UNTAR_DIR

   build_file=$(basename $DCS_BUILD)
   DCS_BUILD=$UNTAR_DIR/$build_file

   build_file=$(basename $REST_BUILD)
   REST_BUILD=$UNTAR_DIR/$build_file

fi

sed -i '/TRAF_BUILD\=/d' $TRAF_CONFIG
echo "export TRAF_BUILD=\"$TRAF_BUILD\"" >> $TRAF_CONFIG
sed -i '/DCS_BUILD\=/d' $TRAF_CONFIG
echo "export DCS_BUILD=\"$DCS_BUILD\"" >> $TRAF_CONFIG
sed -i '/REST_BUILD\=/d' $TRAF_CONFIG
echo "export REST_BUILD=\"$REST_BUILD\"" >> $TRAF_CONFIG

# add the untar directory to the config file
sed -i '/UNTAR_DIR\=/d' $TRAF_CONFIG
echo "export UNTAR_DIR=\"$UNTAR_DIR\"" >> $TRAF_CONFIG
source $TRAF_CONFIG

#=============================================
#Check if doing an upgrade

userForTrafodion=`grep "$TRAF_USER:" /etc/passwd | wc -l`
if [[ $CONFIG_COMPLETE ]]; then
   if [[ "$userForTrafodion" == "0" ]]; then
      UPGRADE_TRAF=false;
   else
      UPGRADE_TRAF=true;
   fi

   #Add upgrade variable to config file
   sudo chmod 777 $TRAF_CONFIG
   sed -i '/UPGRADE_TRAF\=/d' $TRAF_CONFIG
   echo "export UPGRADE_TRAF=\"$UPGRADE_TRAF\"" >> $TRAF_CONFIG
   sudo chmod 777 $TRAF_CONFIG
   source $TRAF_CONFIG
fi

#==============================================
#Check to make sure HBase Version is the correct version and the same
#on all nodes

for node in $NODE_LIST;
do
   ssh $node -n 'echo "exit" | hbase shell &> $HOME/hbaseVersion.txt'
   hbaseInstalled=$(ssh -n $node grep "Version" $HOME/hbaseVersion.txt | wc -l)
   if [[ $hbaseInstalled -eq "0" ]]; then
      echo "***ERROR: Hbase version could not be determined on $node"
      echo "***ERROR: Please check that HBase is installed on all nodes."
      exit -1
   fi

   version=$(ssh -n $node grep "Version" $HOME/hbaseVersion.txt | sed 's/-.*//' | awk {'print$2'})

   if [[ $HADOOP_TYPE == "hortonworks" ]]; then
      if [[ $CDH_5_3_HDP_2_2_SUPPORT == "N" ]]; then
         supportedHBase=$(echo $version | grep "0.98.0.2" | wc -l)
         if [[ $supportedHBase -eq "0" ]]; then
            echo "***ERROR: HBase installed must be 0.98.0.2 on node $node"
            exit -1
         fi
      else
         supportedHBase=$(echo $version | grep "0.98.4" | wc -l)
         if [[ $supportedHBase -eq "0" ]]; then
            echo "***ERROR: HBase installed must be 0.98.4 on node $node"
            exit -1
         fi
      fi
   fi
   if [[ $HADOOP_TYPE == "cloudera" ]]; then
      if [[ $CDH_5_3_HDP_2_2_SUPPORT == "N" ]]; then
         supportedHBase=$(echo $version | grep "0.98.1" | wc -l)
         if [[ $supportedHBase -eq "0" ]]; then
            echo "***ERROR: HBase installed must be 0.98.1 on node $node"
            exit -1
         fi
      else
        supportedHBase=$(echo $version | grep "0.98.6" | wc -l)
         if [[ $supportedHBase -eq "0" ]]; then
            echo "***ERROR: HBase installed must be 0.98.6 on node $node"
            exit -1
         fi
      fi
   fi
   hbaseVersion=$(echo "${version#*.}"| cut -f1 -d".")
   if [[ $hbaseVersion -lt "98" ]]; then
      echo "***ERROR: HBase installed must be 0.98"
      exit -1
   fi
done
hbaseVersion=$(grep "Version" $HOME/hbaseVersion.txt | sed 's/-.*//' | awk {'print$2'})
HBASE=$(curl -su $ADMIN:$PASSWORD http://$URL/api/v1/clusters/$CLUSTER_NAME/services | grep name | grep hbase | sed -e 's@[,]@@'g | awk '{print $3}' | sed "s/\"//g")


sudo chmod 777 $TRAF_CONFIG
sed -i '/hbaseVersion\=/d' $TRAF_CONFIG
echo "export hbaseVersion=\"$hbaseVersion\"" >> $TRAF_CONFIG
sed -i '/HBASE\=/d' $TRAF_CONFIG
echo "export HBASE=\"$HBASE\"" >> $TRAF_CONFIG
sudo chmod 777 $TRAF_CONFIG
sudo rm -rf hbaseVersion.txt

trafodionFullName=$(basename $TRAF_BUILD)
echo "***DEBUG: trafodionFullName=$trafodionFullName"
#Debug version
trafodionVersion=$(echo $trafodionFullName | sed 's/trafodion_server-//' | sed 's/-debug//' | sed 's/.tgz//')

echo "***INFO: Trafodion version = $trafodionVersion"

#==============================================
# Determine Trafodion version by parsing it
# from the trafodion_server tar file.
# Determine Java version

cd $UNTAR_DIR

# clean up any old files left lying around
sudo rm -rf $UNTAR_DIR/export/lib 2>/dev/null

# extract the hbase-trx jar file (and others) from the build tar file
sudo tar -xzf $TRAF_BUILD export/lib 2>&1 | tee -a $INSTALL_LOG
if [ ${PIPESTATUS[0]} != 0 ]; then
    echo "***ERROR: unable to find export/lib/ directory in $TRAF_BUILD" 2>&1 | tee -a $INSTALL_LOG
    exit -1
fi

# parse the version string from the trafodion_server-n.n.n.jar filename
traf_version=$(sudo ls $UNTAR_DIR | \
               grep --max-count=1 trafodion_server | \
               sed -e 's@trafodion_server-\([0-9*].*\).tgz@\1@')
TRAF_VERSION=$(echo $traf_version | sed -e 's@\([0-9*].*\)-debug@\1@')

# Detect java version
java_exec=$(ps -C java --noheader -o pid,user,cmd \
   | grep --max-count=1 " hbase" \
   | awk '{print $3}' )
echo "***DEBUG: HBase's java_exec=$java_exec"
if [ -z $java_exec ]; then
   echo "***ERROR: Unable to determine HBase's Java version"
   echo "***ERROR: Check that HBase is up on node $HOSTNAME"
   exit -1
fi

JAVA_VERSION=$($java_exec -version 2>&1)
JAVA_VERSION=$(echo $JAVA_VERSION \
   | grep version \
   | awk '{print $3}' \
   | sed -e 's@"@@g')

# test for java 1.7
java_17=$(echo $JAVA_VERSION | grep "1.7." | wc -l)

# if not java 1.7 then test for java 1.6
if [ "$java_17" == "0" ]; then
   java_16=$(echo $JAVA_VERSION | grep "1.6." | wc -l)

   # if not 1.7 nor 1.6 then we have a problem
   if [ "$java_16" == "0" ]; then
      echo "***ERROR: Unsupported HBase java version.  HBase java version must be 1.6 or 1.7"
      exit -1
   else
      JAVA_VERSION="1.6"
   fi
else
   JAVA_VERSION="1.7"
fi

sudo chmod 777 $TRAF_CONFIG
sed -i '/TRAF_VERSION\=/d' $TRAF_CONFIG
echo "export TRAF_VERSION=\"$TRAF_VERSION\"" >> $TRAF_CONFIG
sed -i '/JAVA_VERSION\=/d' $TRAF_CONFIG
echo "export JAVA_VERSION=\"$JAVA_VERSION\"" >> $TRAF_CONFIG
sudo chmod 777 $TRAF_CONFIG

#==============================================
# Run trafodion_setup
echo
echo "******************************"
echo " TRAFODION SETUP"
echo "******************************"
echo

if [[ "$UPGRADE_TRAF" == "true" ]]; then
   echo "***INFO: $TRAF_USER username has already been installed." | tee -a $INSTALL_LOG
   echo "***INFO: Skipping Trafodion setup..." | tee -a $INSTALL_LOG

   trafodionRunning=$(sudo su $TRAF_USER --login --command "cstat --no-header | wc -l")

   if [[ "$trafodionRunning" -ne "0" ]]; then
      echo "***ERROR: Trafodion is still running."
      echo -n "Stop Trafodion (Y/N), default is N: "
      read answer

      if [ -z $answer ]; then
         echo "***ERROR: Please stop Trafodion then rerun trafodion_install"
         echo "***ERROR: Change to $TRAF_USER userid then do a 'sqstop'"
         exit -1
      else
         if [[ ${answer^} == "Y" ]]; then
            echo "***INFO: Stopping Trafodion instance."
            sudo su $TRAF_USER --login --command "sqstop"
            sudo su $TRAF_USER --login --command "ckillall"
         else
            echo "***ERROR: Please stop Trafodion then rerun trafodion_install"
            echo "***ERROR: Change to $TRAF_USER user id then do a 'sqstop'"
            exit -1
         fi
      fi

   fi
else
   $TRAF_WORKDIR/installer/traf_setup | tee -a $INSTALL_LOG

   if [ ${PIPESTATUS[0]} != "0" ]; then
      echo "***ERROR: Error while running traf_setup." | tee -a $INSTALL_LOG
      echo "***ERROR: Setup not complete, review logs." | tee -a $INSTALL_LOG
      echo "***ERROR: Exiting...." | tee -a $INSTALL_LOG
      exit -1
   else
      echo "***INFO: Trafodion environment setup completed" | tee -a $INSTALL_LOG
   fi
fi

# create working directories on all nodes
$TRAF_PDSH sudo mkdir -p $TRAF_WORKDIR
$TRAF_PDSH sudo chmod 777 $TRAF_WORKDIR
$TRAF_PDSH sudo mkdir -p $LOCAL_WORKDIR
$TRAF_PDSH sudo chmod 777 $LOCAL_WORKDIR

# copy config file to all nodes
if [ $node_count -ne 1 ]; then
   cp $TRAF_CONFIG $LOCAL_WORKDIR
   pdcp $MY_NODES $LOCAL_WORKDIR/$TRAF_CONFIG_FILE $LOCAL_WORKDIR
   $TRAF_PDSH sudo mkdir -p $TRAF_CONFIG_DIR
   $TRAF_PDSH sudo cp $LOCAL_WORKDIR/$TRAF_CONFIG_FILE $TRAF_CONFIG_DIR
   $TRAF_PDSH sudo chmod 777 $TRAF_CONFIG
fi


#Create sqconfig file if it does not exist

if [[ ! -f $HOME_DIR/$TRAF_USER/sqconfig ]]; then
   $TRAF_WORKDIR/installer/traf_sqconfig
   if [ $? != 0 ]; then
      echo "***ERROR: traf_sqconfig script failed."
      echo "***ERROR: sqconfig file not created."
      exit -1
   fi
fi

#Reserve DCS Ports

echo "***INFO: Reserving DCS ports"
trafodion_ports=${DCS_MASTER_PORT:-37800},${DCS_MASTER_INFO_PORT:-40010}
reservedTrafodionPorts="net.ipv4.ip_local_reserved_ports = ${trafodion_ports}"

for node in $NODE_LIST
do
   redhatSystem=$(ssh $node -n '/sbin/sysctl net.ipv4.ip_local_reserved_ports | wc -l')
   if [[ "$redhatSystem" -ge "1" ]]; then
      reserved_ports=$(ssh $node -n '/sbin/sysctl net.ipv4.ip_local_reserved_ports' | awk -F'=' '{print $2;}' | sed 's/^ //')
      ssh $node -n 'sudo chmod 777 /etc/sysctl.conf'
      ssh $node -n 'sudo sed -i '/net.ipv4.ip_local_reserved_ports""\=*/d' /etc/sysctl.conf'

   if [[ -z "${reserved_ports}" ]]; then
      ssh $node -n 'sudo echo "$reservedTrafodionPorts" >> /etc/sysctl.conf'
   else
      ssh $node -n "sudo echo "$reservedTrafodionPorts","$reserved_ports" >> /etc/sysctl.conf"
   fi
   ssh $node -n 'sudo chmod 644 /etc/sysctl.conf'
   ssh $node -n 'sudo /sbin/sysctl -p'
   else
      echo "***ERROR: Unable to reserve DCS ports (37800 and 40010)."
      exit -1
   fi
done


#>>> we don't really need to copy all this to the trafodion userid's home dir
#Create install directory
pdsh -S $MY_NODES sudo mkdir -p /$HOME_DIR/$TRAF_USER/installer &> /dev/null

#Copy over traf_authentication_config file to all nodes
if [[ "$LDAP_SECURITY" == "Y" ]]; then
   echo "***INFO: Copying $LDAP_AUTH_FILE to Trafodion home dir"

   if [[ "$node_count" -eq "1" ]]; then
      sudo cp -r $LOCAL_WORKDIR/$LDAP_AUTH_FILE $HOME_DIR/$TRAF_USER
      sudo chown trafodion.trafodion $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE
      sudo chmod 750 $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE
      if [[ $LDAP_LEVEL == "2" ]] || [[ $LDAP_LEVEL == "1" ]]; then
         echo "***INFO: Copying $LDAP_CERT to Trafodion home dir"
         sudo cp -r $LDAP_CERT $HOME_DIR/$TRAF_USER
         sudo chown trafodion.trafodion $HOME_DIR/$TRAF_USER/$LDAP_CERT_BASE
         sudo chmod 750 $HOME_DIR/$TRAF_USER/$LDAP_CERT_BASE
      fi
   else
      pdcp $MY_NODES $LOCAL_WORKDIR/$LDAP_AUTH_FILE $HOME
      pdsh $MY_NODES sudo cp -r $HOME/$LDAP_AUTH_FILE $HOME_DIR/$TRAF_USER
      pdsh $MY_NODES sudo chown trafodion.trafodion $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE
      pdsh $MY_NODES sudo chmod 750 $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE
      if [[ $LDAP_LEVEL == "2" ]] || [[ $LDAP_LEVEL == "1" ]]; then
         echo "***INFO: Copying $LDAP_CERT to Trafodion home dir"
         pdcp $MY_NODES $LDAP_CERT $HOME
         pdsh $MY_NODES sudo cp -r $HOME/$LDAP_CERT_BASE $HOME_DIR/$TRAF_USER
         pdsh $MY_NODES sudo chown trafodion.trafodion $HOME_DIR/$TRAF_USER/$LDAP_CERT_BASE
         pdsh $MY_NODES sudo chmod 750 $HOME_DIR/$TRAF_USER/$LDAP_CERT_BASE
      fi
   fi
fi
#=============================================
#Run trafodion_mods

echo
echo "******************************"
echo " TRAFODION MODS"
echo "******************************"
echo

if [[ $HADOOP_TYPE == "cloudera" ]]; then
   echo "***INFO: Cloudera installed will run traf_cloudera_mods98" 2>&1 | tee -a $INSTALL_LOG
   $TRAF_WORKDIR/installer/traf_cloudera_mods98 2>&1 | tee -a $INSTALL_LOG
   if [ ${PIPESTATUS[0]} != "0" ]; then
      echo "***ERROR: traf_cloudera_mods98 exited with error." | tee -a $INSTALL_LOG
      echo "***ERROR: Please check log files." | tee -a $INSTALL_LOG
      echo "***ERROR: Exiting...." | tee -a $INSTALL_LOG
      exit -1
   fi
fi

if [[ $HADOOP_TYPE == "hortonworks" ]]; then
   echo "***INFO: Hortonworks installed will run traf_hortonworks_mods98" 2>&1 | tee -a $INSTALL_LOG
   $TRAF_WORKDIR/installer/traf_hortonworks_mods98 2>&1 | tee -a $INSTALL_LOG
   if [ ${PIPESTATUS[0]}  != "0" ]; then
      echo "***ERROR: traf_hortonworks_mods98 exited with error." | tee -a $INSTALL_LOG
      echo "***ERROR: Please check log files." | tee -a $INSTALL_LOG
      echo "***ERROR: Exiting...." | tee -a $INSTALL_LOG
      exit -1
   fi
fi

echo "***INFO: Trafodion Mods ran successfully."

#==============================================
#Install Trafodion
echo
echo "******************************"
echo " TRAFODION START"
echo "******************************"
echo
sudo mkdir -p $SQ_ROOT
sudo chown $TRAF_USER.trafodion $SQ_ROOT
sudo su $TRAF_USER --command "$TRAF_WORKDIR/installer/traf_start"

if [ ${PIPESTATUS[0]} != "0" ]; then
   echo "***ERROR: Error while running traf_start." | tee -a $INSTALL_LOG
   echo "***ERROR: Setup not complete, review logs." | tee -a $INSTALL_LOG
   echo "***ERROR: Exiting...." | tee -a $INSTALL_LOG
   exit -1
fi

#==============================================
echo
echo "*********************************"
echo " TRAFODION INSTALLATION COMPLETE"
echo "*********************************"
echo