core/sqf/sql/scripts/install_local_hadoop

#!/bin/bash
# @@@ START COPYRIGHT @@@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# @@@ END COPYRIGHT @@@
#
##############################################################################
##
## This sets up a Hadoop/Hive/HBase environment to be used with a trafodion
## workstation development environment. This script is meant for developers
## and has the following characteristics:
##
## - Creates a pseudo-distributed single node installation
## - Sandboxes the installation into a single directory,
##   $TRAF_HOME/sqf/sql/local_hadoop
## - creates some convenience shell scripts in $TRAF_HOME/sqf/sql/scripts
##   to start and stop Hadoop and for some interactive shells
## - Does not require sudo privileges for installing or running Hadoop
## - Can run on non-standard ports, if needed, to be able to run multiple
##   Hadoop/Hive/HBase instances on the same machine
## - Uses a file system directory to store HBase data, not HDFS
## - Uses MySQL as the Hive metastore
## - Creates a TPC-DS sample database in Hive
##
##############################################################################
# Environment variables - optional
#
# MY_LOCAL_SW_DIST - shared location on local network for tar balls for Hadoop, etc.
# - Hadoop
# - HBase
# - Hive
# - MySQL (used as Hive metastore DB)
# - MySQL connector for Java (used by Hive to access metastore)
# - TPC-DS from tpc.org (for Hive sample DB)
# also set http_proxy and ftp_proxy if necessary, to download
# files from repositories on the Internet
#
##############################################################################

function usage {

cat <<EOF

Usage:

$MY_CMD [ -p {<start port num> | rand | fromDisplay} ]
                     [ -y ]
                     [ -n ]
                     [ -v ]

  -p configures non-standard ports, and is one of:
    -p <start port num>     custom cases
    -p rand                 for shared systems, use a random start port number
                            between 9000 and 49000 that is divisible by 200
    -p fromDisplay          if you are running on a VNC session

  -y answers interactive questions implicitly with yes

  -n takes no action, useful with -v

  -v lists the port values used

  See script header for use of optional environment variables.

EOF
}

function check_ssh {
  SSH_FAILED=no
  echo
  echo "Check ssh ${MY_HOST_1} access without a password..."
  # disable any options that would query the user terminal, ask for strict host key checking
  # to force it to fail if something is not quite right with the host key
  ssh -o PasswordAuthentication=no -o KbdInteractiveDevices=none -o StrictHostKeyChecking=yes ${MY_HOST_1} echo "testing ssh ${MY_HOST_1}"
  if [ $? -ne 0 ]; then
    # if we deal with multiple systems, don't wipe out .ssh directory
    # try some simpler measures and give up if those don't work
    echo "Problems encountered with ssh, trying to fix some common issues..."
    if [ -d ~/.ssh ]; then
      grep -q NoHostAuthenticationForLocalhost ~/.ssh/config
      if [ $? -ne 0 ]; then
        # configure ssh to omit host check for localhost
        grep -qi ^host ~/.ssh/config
        if [ $? -eq 0 ]; then
          # make sure this applies to all hosts, even if
          # there are host directives in the file
          echo "host *"  >>~/.ssh/config
        fi
        echo "NoHostAuthenticationForLocalhost=yes" >>~/.ssh/config
        chmod go-w ~/.ssh/config
      fi

      if [ ${MY_HOST_1} != "localhost" ]; then
        # remove existing entries for the host from known_hosts
        if [ -f ~/.ssh/known_hosts ]; then
          ed ~/.ssh/known_hosts <<EOF
g/^${MY_HOST_1}/d
w
EOF
        fi
        # add an entry for our host to known_hosts
        echo "${MY_HOST_1}" `cat /etc/ssh/ssh_host_rsa_key.pub` >>~/.ssh/known_hosts
        chmod 644 ~/.ssh/known_hosts
      fi

      # check whether doing ssh-add will fix the problem
      ps -aef | grep ${USER} | grep ssh-agent | grep -v grep >/dev/null
      if [ $? -eq 0 ]; then
        # ssh-agent is running, tell it to use the new key
        ssh-add
      fi

      # now try once more (with regular host key checking)
      ssh -o PasswordAuthentication=no -o KbdInteractiveDevices=none ${MY_HOST_1} echo "testing ssh ${MY_HOST_1}"
      if [ $? -ne 0 ]; then
        SSH_FAILED=yes
      fi
    else
      SSH_FAILED=yes
    fi
  fi

  if [ $SSH_FAILED = yes ]; then
    # A few initial steps that are required:
    cat <<EOF
Please make sure you can do ssh ${MY_HOST_1} without having to enter a password
(this is a one-time setup):

cd
rm -rf .ssh
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 644 ~/.ssh/authorized_keys
ssh-add

EOF

    if [ -z "$MY_IMPLICIT_Y" ]; then
      echo "Would you like to configure ssh to ${MY_HOST_1}? This will wipe out your existing"
      echo "~/.ssh directory and you will lose existing private key files."
      echo " "
      echo -n "Enter y/n (n): "

      read YN
    else
      # user already enabled on command line
      YN=$MY_IMPLICIT_Y
    fi

    if [ "$YN" = "y" -o "$YN" = "Y" ]; then
      echo "Setting up public/private key pair for connection to ${MY_HOST_1}..."
      echo "Saving the old ~/.ssh directory to ~/.ssh-renamed, in case this is not what you wanted"
      cd
      rm -rf .ssh-renamed
      mv -f .ssh .ssh-renamed
      # generate an RSA public/private key pair without a passphrase
      ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

      # copy public key to authorized_keys and set permissions correctly
      chmod 600 ~/.ssh/id_rsa
      cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
      chmod 644 ~/.ssh/authorized_keys

      # avoid the interactive question to authenticate localhost
      # as a valid host (this might cause problems when running
      # this script in batch mode)
      echo "${MY_HOST_1}" `cat /etc/ssh/ssh_host_rsa_key.pub` >~/.ssh/known_hosts
      chmod 644 ~/.ssh/known_hosts

      # disable checks for "localhost", so we can use this on multiple machines
      echo "NoHostAuthenticationForLocalhost=yes" >>~/.ssh/config
      chmod 644 ~/.ssh/config

      ps -aef | grep ${USER} | grep ssh-agent | grep -v grep >/dev/null
      if [ $? -eq 0 ]; then
        # ssh-agent is running, tell it to use the new key
        ssh-add
      fi

      # now try once more (with regular host key checking)
      ssh -o PasswordAuthentication=no -o KbdInteractiveDevices=none ${MY_HOST_1} echo "testing ssh ${MY_HOST_1}"
      if [ $? -ne 0 ]; then
        SSH_FAILED=yes
      fi
    else
      echo "================================================="
      echo "exiting, set up ssh access on your own,"
      echo "remove ${MY_SW_ROOT} and rerun this"
      echo "installation script, $0"
      echo "================================================="
      exit 1
    fi
  else
    echo "Check ssh ${MY_HOST_1} access without a password succeeded."
  fi
  # end of ssh checking
}

function listports {
  # Report values selected for ports
  VARS_FOR_PORTS="
	MY_DCS_MASTER_INFO_PORT
	MY_DCS_MASTER_PORT
	MY_DCS_SERVER_INFO_PORT
	MY_HADOOP_DN_HTTP_PORT_NUM
	MY_HADOOP_DN_IPC_PORT_NUM
	MY_HADOOP_DN_PORT_NUM
	MY_HADOOP_HDFS_PORT_NUM
	MY_HADOOP_JOB_TRACKER_HTTP_PORT_NUM
	MY_HADOOP_NN_HTTP_PORT_NUM
	MY_HADOOP_SECONDARY_NN_PORT_NUM
	MY_HADOOP_SHUFFLE_PORT_NUM
	MY_HADOOP_TASK_TRACKER_PORT_NUM
	MY_HBASE_MASTER_INFO_PORT_NUM
	MY_HBASE_MASTER_PORT_NUM
	MY_HBASE_REGIONSERVER_INFO_PORT_NUM
	MY_HBASE_REGIONSERVER_PORT_NUM
	MY_HBASE_REST_PORT_NUM
	MY_HBASE_ZOOKEEPER_LEADERPORT_NUM
	MY_HBASE_ZOOKEEPER_PEERPORT_NUM
	MY_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT_NUM
	MY_REST_SERVER_PORT
	MY_REST_SERVER_SECURE_PORT
	MY_SQL_PORT_NUM
	MY_YARN_ADMIN_PORT_NUM
	MY_YARN_HTTP_PORT_NUM
	MY_YARN_LOCALIZER_PORT_NUM
	MY_YARN_NM_PORT_NUM
	MY_YARN_RESMAN_PORT_NUM
	MY_YARN_SCHED_PORT_NUM
	MY_YARN_TRACKER_PORT_NUM
        MY_DB_MGR_PORT
        MY_DB_MGR_SECURE_PORT"

  for AVAR in $VARS_FOR_PORTS; do
    AVALUE="$(eval "echo \$$AVAR")"
    printf '%s=%s\n' $AVAR $AVALUE
  done
  echo
}

# global config

# directory on local disk that belongs to me
MY_SW_PARENT=$TRAF_HOME/sql

# name of subdirectory to install Hadoop/Hive/MySQL in
MY_SW_ROOT=$MY_SW_PARENT/local_hadoop

# directory where convenience scripts will be located
MY_SW_SCRIPTS_DIR=$MY_SW_PARENT/scripts

MYSQL_HOME=${MY_SW_ROOT}/mysql
YARN_HOME=${MY_SW_ROOT}/hadoop
HIVE_HOME=${MY_SW_ROOT}/hive
HBASE_HOME=${MY_SW_ROOT}/hbase

  # shared location on local network for tar balls for Hadoop, etc.
  # - Hadoop
  # - HBase
  # - Hive
  # - MySQL (used as Hive metastore DB)
  # - MySQL connector for Java (used by Hive to access metastore)
  # - TPC-DS from tpc.org (for Hive sample DB)
  # ------------- please customize this line ----------------------
  if [ -z "$MY_LOCAL_SW_DIST" ]; then
    MY_LOCAL_SW_DIST=/add_your_local_shared_folder_here
  fi

  # also set http_proxy and ftp_proxy if necessary, to download
  # files from repositories on the Internet


# hosts, only localhost or the actual DNS name of the local host are supported for now
############################################
MY_HOST_1=localhost
# this may get changed below to a DNS name, with -d command line option

MY_LOG_FILE=${MY_SW_ROOT}/log/install_local_hadoop_$(date +%F_%T).log

# multiple hosts not yet supported

# locations for storing data and metadata
##########################################
MY_DATA_DIR=${MY_SW_ROOT}/data
MY_SQL_DATA_DIR=${MY_DATA_DIR}/mysql
MY_DERBY_DATA_DIR=${MY_DATA_DIR}/derby
MY_HADOOP_DATA_DIR=${MY_DATA_DIR}/hadoop
MY_HIVE_DATA_DIR=${MY_DATA_DIR}/hive
MY_HBASE_DATA_DIR=${MY_DATA_DIR}/hbase
MY_ZOOKEEPER_DATA_DIR=${MY_DATA_DIR}/zookeeper

# Administrator and Hive user name for MySQL installation
MY_SQL_ADMIN=root
MY_SQL_USER=$MY_SQL_ADMIN
# generate a random password to use for MySQL (no spaces or special characters)
MY_SQL_ADMIN_PASSWD=p${RANDOM}${RANDOM}
MY_SQL_USER_PASSWD=$MY_SQL_ADMIN_PASSWD
# database name for Hive metastore
MY_SQL_METASTORE_DB=metastore

# MySQL configuration file
MY_SQL_CONFIG_FILE=${MYSQL_HOME}/my.cnf

MY_CMD=$0
MY_IMPLICIT_Y=
NOACTION=
VERBOSE=
MISSING_P_VAL="ERROR: -p option should be followed by <start port num>, rand, fromDisplay or available"

# process command line arguments
################################

while [ $# -gt 0 ];
do
  case $1 in
-p) shift
    if [[ -z "$1" ]]; then
      echo "$MISSING_P_VAL"
      exit 1
    fi
    MY_START_PORT=$1
    if [ $MY_START_PORT != "rand" -a $MY_START_PORT != "fromDisplay" -a $MY_START_PORT != "available" ]; then
      test $MY_START_PORT -gt 0 >/dev/null 2>&1
      if [ $? -ne 0 ]; then
        echo "$MISSING_P_VAL"
        exit 1
      fi
    fi
    ;;
-y) MY_IMPLICIT_Y="y"
    ;;
-d) # use DNS name instead of "localhost" (option is not currently recommended)
    MY_HOST_1=`uname -a | cut -f 2 -d ' '`
    ;;
-h|-help)
    usage
    exit 0
    ;;
-n) NOACTION="y"
    ;;
-v) VERBOSE="y"
    ;;
*)  echo "ERROR: Unexpected argument $1"
    echo
    cat <<EOF
Syntax: $0 [ -p [<starting port num> | rand | fromDisplay | available] ]  [-y]  [-n] [-v]
EOF
    exit 1
    ;;
  esac
  shift
done

# port numbers used
####################

# From http://blog.cloudera.com/blog/2009/08/hadoop-default-ports-quick-reference/
# and http://hbase.apache.org/book/config.files.html

USE_DEFAULT_PORTS=no
# To be done later, use separate ports to allow multiple
# Hadoop instances on the same Linux node
if [ -z "$MY_START_PORT" ]; then
  USE_DEFAULT_PORTS=yes
  MY_START_PORT=50000
else
  if [ $MY_START_PORT == "fromDisplay" ]; then
    # display :34.0 would result in starting port 53400 (instead of default 50000 range)
    MY_START_PORT=${DISPLAY/*:/}
    MY_START_PORT=${MY_START_PORT/.*/}
    # Pick a port range starting at 24000 + 200 * display number.
    # Choose a display number of up to 42 to stay below the
    # ephemeral port range which typically starts at 32768.
    MY_START_PORT=`expr 24000 + $MY_START_PORT '*' 200`
  elif [ $MY_START_PORT == "rand" ]; then
    # pick a random number between 12000 and 24000 that is divisible by 200
    MY_START_PORT=`expr $RANDOM '%' 60 '*' 200 + 12000`
  elif [ $MY_START_PORT == "available" ]; then

     portMsg=`python findPort.py`

echo $portMsg
     missingPort=`echo $portMsg | cut -d':' -f 2-`

     if [[ "$missingPort" == "" ]]; then
        echo "No free port available, exit"
        exit 1
     fi

     ports=`echo $portMsg | cut -d':' -f 2-2`
     port_arr=($ports)

     port_arr1=()

     # filter out any ports that are not between 12000 and 24000 that "-p rand" uses
     # this is to assure consistency between this option and the rand option.
     for elem in "${port_arr[@]}"
     do
       if [ $elem -ge 12000 -a $elem -le 24000 ]; then
         port_arr1[${#port_arr1[@]}]=$elem
       fi
     done

     num_ports=${#port_arr1[@]}

     echo "Available ports detected for use: ${port_arr1[*]}"
     echo "Total number of free ports: $num_ports"

     if [ $num_ports -eq 0 ]; then
       echo "No free ports available, exit."
       exit 1
     fi

     random_port_index=`python -c "import random; print random.randint(0,${#port_arr1[@]}-1)"`
     echo "Port index randomely chosen: $random_port_index"

     MY_START_PORT="${port_arr1[$random_port_index]}"
     echo "Port chosen: $MY_START_PORT"
  fi
  echo "# Using non-standard port range from MY_START_PORT env var: $MY_START_PORT..."
fi

# assign ports with defaults outside the range of 50000-50199
if [ $USE_DEFAULT_PORTS == 'yes' ]; then
  # fs.default.name
  MY_HADOOP_HDFS_PORT_NUM=9000
  # mapreduce.shuffle.port
  MY_HADOOP_SHUFFLE_PORT_NUM=8080
  # yarn.resourcemanager.address
  MY_YARN_RESMAN_PORT_NUM=8032
  # yarn.resourcemanager.scheduler.address
  MY_YARN_SCHED_PORT_NUM=8030
  # yarn.resourcemanager.webapp.address
  MY_YARN_HTTP_PORT_NUM=8088
  # yarn.resourcemanager.resource-tracker.address
  MY_YARN_TRACKER_PORT_NUM=8031
  # yarn.resourcemanager.admin.address
  MY_YARN_ADMIN_PORT_NUM=8033
  # yarn.nodemanager.localizer.address
  MY_YARN_LOCALIZER_PORT_NUM=8040
  # yarn.nodemanager.webapp.address
  MY_YARN_NM_PORT_NUM=8041
  # unique port # for MySQL (don't use default of 3306, often used already)
  MY_SQL_PORT_NUM=3346
  # hbase.master.port
  MY_HBASE_MASTER_PORT_NUM=60000
  # hbase.master.info.port
  MY_HBASE_MASTER_INFO_PORT_NUM=60010
  # hbase.regionserver.port
  MY_HBASE_REGIONSERVER_PORT_NUM=60020
  # hbase.regionserver.info.port
  MY_HBASE_REGIONSERVER_INFO_PORT_NUM=60030
  # hbase.zookeeper.peerport
  MY_HBASE_ZOOKEEPER_PEERPORT_NUM=2888
  # hbase.zookeeper.leaderport
  MY_HBASE_ZOOKEEPER_LEADERPORT_NUM=3888
  # hbase.zookeeper.property.clientPort
  MY_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT_NUM=2181
  # hbase.rest.port
  MY_HBASE_REST_PORT_NUM=8080
  # dcs.master.port (range of port numbers, one per server)
  MY_DCS_MASTER_PORT=23400
  # dcs.master.info.port
  MY_DCS_MASTER_INFO_PORT=24400
  # dcs.server.info.port
  MY_DCS_SERVER_INFO_PORT=24410
  # trafodion.rest.port
  MY_REST_SERVER_PORT=4200
  # trafodion.rest.https.port
  MY_REST_SERVER_SECURE_PORT=4201
  # database.mgr.port
  MY_DB_MGR_PORT=4205
  # database.mgr.secure.port
  MY_DB_MGR_SECURE_PORT=4206
  # drill.exec.http.port
  MY_DRILL_INFO_PORT_NUM=8047
  # drill.exec.rpc.user.server.port
  MY_DRILL_RPC_PORT_NUM=31010
  # drill.exec.rpc.bit.server.port
  MY_DRILL_BIT_PORT_NUM=31011
else
  # fs.default.name

  MY_HADOOP_HDFS_PORT_NUM=$MY_START_PORT
  MY_HADOOP_SHUFFLE_PORT_NUM=`expr $MY_START_PORT + 62`
  MY_YARN_RESMAN_PORT_NUM=`expr $MY_START_PORT + 132`
  MY_YARN_SCHED_PORT_NUM=`expr $MY_START_PORT + 130`
  MY_YARN_HTTP_PORT_NUM=`expr $MY_START_PORT + 188`
  MY_YARN_TRACKER_PORT_NUM=`expr $MY_START_PORT + 131`
  MY_YARN_ADMIN_PORT_NUM=`expr $MY_START_PORT + 133`
  MY_YARN_LOCALIZER_PORT_NUM=`expr $MY_START_PORT + 140`
  MY_YARN_NM_PORT_NUM=`expr $MY_START_PORT + 141`
  MY_HBASE_MASTER_PORT_NUM=`expr $MY_START_PORT + 160`
  MY_HBASE_MASTER_INFO_PORT_NUM=`expr $MY_START_PORT + 161`
  MY_HBASE_REGIONSERVER_PORT_NUM=`expr $MY_START_PORT + 162`
  MY_HBASE_REGIONSERVER_INFO_PORT_NUM=`expr $MY_START_PORT + 163`
  MY_HBASE_ZOOKEEPER_PEERPORT_NUM=`expr $MY_START_PORT + 167`
  MY_HBASE_ZOOKEEPER_LEADERPORT_NUM=`expr $MY_START_PORT + 168`
  MY_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT_NUM=`expr $MY_START_PORT + 170`
  MY_HBASE_REST_PORT_NUM=`expr $MY_START_PORT + 171`
  # unique port # for MySQL (default is 3306)
  MY_SQL_PORT_NUM=`expr $MY_START_PORT + 46`
  # MY_DCS_MASTER_PORT is a range of ports, one per server
  # (see "server" file written below for how many are configured)
  MY_DCS_MASTER_PORT=`expr $MY_START_PORT + 172`
  MY_DCS_MASTER_INFO_PORT=`expr $MY_START_PORT + 181`
  MY_DCS_SERVER_INFO_PORT=`expr $MY_START_PORT + 182`
  MY_REST_SERVER_PORT=`expr $MY_START_PORT + 193`
  MY_REST_SERVER_SECURE_PORT=`expr $MY_START_PORT + 194`
  MY_DB_MGR_PORT=`expr $MY_START_PORT + 195`
  MY_DB_MGR_SECURE_PORT=`expr $MY_START_PORT + 196`

  # Apache Drill, embedded configuration
  MY_DRILL_INFO_PORT_NUM=`expr $MY_START_PORT + 147`
  MY_DRILL_RPC_PORT_NUM=`expr $MY_START_PORT + 148`
  MY_DRILL_BIT_PORT_NUM=`expr $MY_START_PORT + 149`
fi

# handle ports in the range of 50000 to 50199

# in hdfs-site.xml (setting any of these to 0 means start on a free port):

# dfs.http.address	        50070	 dfs namenode web ui
MY_HADOOP_NN_HTTP_PORT_NUM=`expr $MY_START_PORT + 70`

# dfs.secondary.http.address	50090	 The secondary namenode http server
MY_HADOOP_SECONDARY_NN_PORT_NUM=`expr $MY_START_PORT + 90`

# dfs.datanode.address	        50010	 datanode server
MY_HADOOP_DN_PORT_NUM=`expr $MY_START_PORT + 10`

# dfs.datanode.http.address	50075	 datanode http server
MY_HADOOP_DN_HTTP_PORT_NUM=`expr $MY_START_PORT + 75`

# dfs.datanode.ipc.address	50020	 datanode ipc server
MY_HADOOP_DN_IPC_PORT_NUM=`expr $MY_START_PORT + 20`

# in mapred-site.xml:

# mapred.job.tracker.http.address (default 50030)
MY_HADOOP_JOB_TRACKER_HTTP_PORT_NUM=`expr $MY_START_PORT + 30`

# mapred.task.tracker.http.address (default 50060)
MY_HADOOP_TASK_TRACKER_PORT_NUM=`expr $MY_START_PORT + 60`

# others, left as default for now:

# dfs.backup.address (50100)
# dfs.backup.http.address (50105)
# mapred.job.tracker (???)

listports > $MY_SW_SCRIPTS_DIR/sw_ports

if [[ -n "$VERBOSE" ]]; then
  listports
fi
if [[ -n "$NOACTION" ]]; then
  exit 0
fi


# Specify mirrors and versions of needed components
#####################################################

# See JIRA TRAFODION-1512 that will eliminate this check
  
HADOOP_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5
HADOOP_TAR=hadoop-2.6.0-cdh5.7.1.tar.gz
if [[ "$HBASE_DISTRO" = "CDH5.5" ]]; then
    HADOOP_TAR=hadoop-2.6.0-cdh5.5.1.tar.gz
fi
if [[ "$HBASE_DISTRO" = "CDH5.4" ]]; then
    HADOOP_TAR=hadoop-2.6.0-cdh5.4.4.tar.gz
fi

if [[ "$HBASE_DISTRO" = "HDP" ]]; then
    HADOOP_TAR=hadoop-2.7.1.2.3.2.0-2950.tar.gz
fi
if [[ "$HBASE_DISTRO" =~ "APACHE" ]]; then
    HADOOP_MIRROR_URL=https://archive.apache.org/dist/hadoop/common/hadoop-2.6.0/
    HADOOP_TAR=hadoop-2.6.0.tar.gz
fi

# Alternative: Use MariaDB (not validated)
# MARIADB_MIRROR_URL=https://downloads.mariadb.org/f/mariadb-5.5.29/kvm-bintar-hardy-amd64/mariadb-5.5.29-linux-x86_64.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
# MARIADB_TAR=mariadb-5.5.29-linux-x86_64.tar.gz

MYSQL_MIRROR_URL=http://cdn.mysql.com/archives/mysql-5.6
MYSQL_TAR=mysql-5.6.10-linux-glibc2.5-x86_64.tar.gz

MYSQL_JDBC_URL=http://cdn.mysql.com/archives/mysql-connector-java-5.1
MYSQL_JDBC_TAR=mysql-connector-java-5.1.23.tar.gz

HIVE_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5
HIVE_PREFIX=hive-${HIVE_DEP_VER_CDH}
if [[ "$HBASE_DISTRO" =~ "APACHE" ]]; then
    HIVE_MIRROR_URL=https://archive.apache.org/dist/hive/hive-${HIVE_DEP_VER_APACHE}
    HIVE_PREFIX=apache-hive-${HIVE_DEP_VER_APACHE}-bin
fi
if [[ "$HBASE_DISTRO" = "HDP" ]]; then
    HIVE_MIRROR_URL=https://archive.apache.org/dist/hive/hive-${HIVE_DEP_VER_HDP}
    HIVE_PREFIX=apache-hive-${HIVE_DEP_VER_HDP}-bin
fi
HIVE_TAR=${HIVE_PREFIX}.tar.gz

HBASE_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5
HBASE_TAR=hbase-${HBASE_DEP_VER_CDH}.tar.gz
if [[ "$HBASE_DISTRO" = "HDP" ]]; then
    HBASE_MIRROR_URL=https://archive.apache.org/dist/hbase/${HBASE_DEP_VER_HDP}
    HBASE_TAR=hbase-${HBASE_DEP_VER_HDP}-bin.tar.gz
fi
if [[ "$HBASE_DISTRO" =~ "APACHE" ]]; then
    HBASE_MIRROR_URL=https://archive.apache.org/dist/hbase/${HBASE_DEP_VER_APACHE}
    HBASE_TAR=hbase-${HBASE_DEP_VER_APACHE}-bin.tar.gz
fi

echo "Checking for existing Hadoop processes..."
if [ `netstat -anl | grep ${MY_HADOOP_JOB_TRACKER_HTTP_PORT_NUM} | grep -w LISTEN | wc -l` -gt 0 -o \
     `netstat -anl | grep ${MY_HADOOP_NN_HTTP_PORT_NUM} | grep -w LISTEN | wc -l` -gt 0 ]; then
  echo '**** ERROR:'
  echo "A process is already listening to port ${MY_HADOOP_JOB_TRACKER_HTTP_PORT_NUM} or ${MY_HADOOP_NN_HTTP_PORT_NUM}."
  echo "This could be your own HDFS web interface or that of someone else."
  echo "Please shut Hadoop down first or switch to another machine."
  if [ -n "$DISPLAY" ]; then
    echo "Alternatively, use non-standard ports with this option:"
    echo "$MY_CMD -p fromDisplay"
  fi
  exit 1
fi

# check for missing tpcds_kit.zip file
INSTALL_TPCDS=1
install_hadoop_regr_test_env --check
if [ $? -ne 0 ]; then
  INSTALL_TPCDS=0
fi

if [ -d "$MY_SW_ROOT" ]; then
  echo "$MY_SW_ROOT already exists, skipping initial steps..."
else
  echo
  echo "Creating common directory $MY_SW_ROOT..."
  #####################################################

  mkdir $MY_SW_ROOT
  cd $MY_SW_ROOT
  mkdir log
  mkdir -p traf_conf
  cp -rf  $TRAF_HOME/conf/* traf_conf/

  cat <<-EOF > traf_conf/sqconfig
	begin node
	node-id=0;node-name=$(hostname -f);cores=0-3;processors=1;roles=connection,aggregation,storage
	end node
	
	begin name-server
	nodes=0
	end name-server
	
	begin overflow
	hdd $TRAF_VAR
	#ssd /ssd/directory
	end overflow
	EOF

  echo
  echo "Checking Java version..."

  # check JAVA_HOME and Java version (1.6 or 1.7)
  if [ -n $JAVA_HOME ]; then
    if [ `expr "${JAVA_HOME}" : ".*_32"` -gt 0 ]; then
      echo "Using a 32 bit Java environment, JAVA_HOME=${JAVA_HOME}"
      echo -n "This might not be optimal. Ok to switch to a 64 bit Java environment? Enter y/n (n):"

      read YN

      if [ "$YN" == "y" -o "$YN" == "Y" ]; then
        unset JAVA_HOME
      else
        echo "Ok, continuing with 32 bit Java..."
      fi
    fi
    if [ -n "$JAVA_HOME" ]; then
      echo "Picked up JAVA_HOME=${JAVA_HOME} from the environment..."
    fi
  fi

  if [ -z "$JAVA_HOME" ]; then
    echo "Trying to determine JAVA_HOME..."

    JAVA_HOME_CANDIDATES="\
         /opt/home/tools/jdk1.6.*_64 \
         /opt/home/tools/jdk1.7.*_64"

    # Add the directory of the java executable in the path to the candidates
    JAVA_EXE=`which java`
    if [ $? -eq 0 ]; then
      # follow symbolic links until we reach the actual file
      while [ -L $JAVA_EXE ]; do
        JAVA_EXE=`readlink $JAVA_EXE`
      done
      JAVA_DIR=`dirname $JAVA_EXE`
      JAVA_HOME_CANDIDATES="$JAVA_HOME_CANDIDATES $JAVA_DIR"
    fi
    cd $MY_SW_ROOT

    # loop through candidates, use the last one that has a
    # java executable in it (preference for later versions and path)
    for c in $JAVA_HOME_CANDIDATES
    do
      if [ -x $c/bin/java ]; then
        JAVA_HOME=$c
      fi
    done

    echo "Setting JAVA_HOME=${JAVA_HOME}"
  fi

  JAVA_EXE=$JAVA_HOME/bin/java
  if [ -z "$JAVA_HOME" -o \( ! -x $JAVA_EXE \) ]; then
    echo '**** ERROR:'
    echo "JAVA_HOME environment variable $JAVA_HOME doesn't point to a java executable, exiting..."
    exit 1
  fi

  $JAVA_EXE -version
  (${JAVA_EXE} -version 2>&1) | grep '1.6' >/dev/null
  if [ $? -ne 0 ]; then
    (${JAVA_EXE} -version 2>&1) | grep '1.7' >/dev/null
    if [ $? -ne 0 ]; then
      (${JAVA_EXE} -version 2>&1) | grep '1.8' >/dev/null
      if [ $? -ne 0 ]; then
        echo '**** ERROR:'
        cat <<EOF
        Please make sure you are using the Java 1.6, 1.7 or 1.8 SDK.
        Otherwise, download it into ${MY_SW_ROOT}, extract it and
        make a symbolic link ${MY_SW_ROOT}/java that points  to it and
        export JAVA_HOME=${MY_SW_ROOT}/java
EOF
        echo exiting...
        exit 1
      else
       echo "Java version 1.8 is ok"
      fi
    else
      echo "Java version 1.7 is ok"
    fi
  else
    echo "Java version 1.6 is ok"
  fi

  # end of check Java version

  echo
  echo "Creating some convenience shell scripts in $MY_SW_SCRIPTS_DIR..."
  if [ ! -d $MY_SW_SCRIPTS_DIR ]; then
    mkdir $MY_SW_SCRIPTS_DIR
  fi

  # shell script to set up common environment variables
  cat <<EOF >$MY_SW_SCRIPTS_DIR/sw_env.sh
# Basic environment variables for Trafodion/Hadoop/Hive/HBase/MySQL setup
export JAVA_HOME=${JAVA_HOME}
export JAVA_LIBRARY_PATH=\${LD_LIBRARY_PATH}
export MY_SW_SCRIPTS_DIR=${MY_SW_SCRIPTS_DIR}
export MY_SW_ROOT=${MY_SW_ROOT}
export MYSQL_HOME=${MYSQL_HOME}
export YARN_HOME=${YARN_HOME}
export HIVE_HOME=${HIVE_HOME}
export HBASE_HOME=${HBASE_HOME}
export HADOOP_PID_DIR=${MY_SW_ROOT}/tmp
export HBASE_PID_DIR=${MY_SW_ROOT}/tmp
export MY_HADOOP_HDFS_PORT_NUM=${MY_HADOOP_HDFS_PORT_NUM}
export MY_HBASE_ZOOKEEPER_PEERPORT_NUM=${MY_HBASE_ZOOKEEPER_PEERPORT_NUM}
export MY_HBASE_ZOOKEEPER_LEADERPORT_NUM=${MY_HBASE_ZOOKEEPER_LEADERPORT_NUM}
export MY_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT_NUM=${MY_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT_NUM}
export MY_ZOOKEEPER_DATA_DIR=${MY_ZOOKEEPER_DATA_DIR}
export MY_DCS_MASTER_PORT=${MY_DCS_MASTER_PORT}
export MY_DCS_MASTER_INFO_PORT=${MY_DCS_MASTER_INFO_PORT}
export MY_DCS_SERVER_INFO_PORT=${MY_DCS_SERVER_INFO_PORT}
export MY_REST_SERVER_PORT=${MY_REST_SERVER_PORT}
export MY_REST_SERVER_SECURE_PORT=${MY_REST_SERVER_SECURE_PORT}
export MY_DB_MGR_PORT=${MY_DB_MGR_PORT}
export MY_DB_MGR_SECURE_PORT=${MY_DB_MGR_SECURE_PORT}
export MY_DRILL_INFO_PORT_NUM=${MY_DRILL_INFO_PORT_NUM}
export MY_DRILL_RPC_PORT_NUM=${MY_DRILL_RPC_PORT_NUM}
export MY_DRILL_BIT_PORT_NUM=${MY_DRILL_BIT_PORT_NUM}
export ZOOKEEPER_NODES=localhost
export ZOOKEEPER_PORT=${MY_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT_NUM}
export TRAF_CLUSTER_ID=1
export TRAF_CLUSTER_NAME=$USER
export TRAF_INSTANCE_ID=1
export TRAF_INSTANCE_NAME=TRAFODION
export TRAF_ROOT_ZNODE=/trafodion

# HBase heap size is in MB
export HBASE_HEAPSIZE=2048
EOF

  # now source in this script
  . $MY_SW_SCRIPTS_DIR/sw_env.sh

  ####################################
  # scripts to start/stop environment
  ####################################
  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstartall
#!/bin/bash
echo "Starting Hadoop, MySQL, HBase..."
cd ${MY_SW_SCRIPTS_DIR}
./swstarthadoop
./swstartmysql
./swstarthbase
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstopall
#!/bin/bash
echo "Stopping Hadoop, MySQL, HBase..."
cd ${MY_SW_SCRIPTS_DIR}
./swstophbase
./swstophadoop
./swstopmysql
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstarthadoop
#!/bin/bash
echo "Starting Hadoop..."
cd ${MY_SW_ROOT}
. $MY_SW_SCRIPTS_DIR/sw_env.sh
./hadoop/sbin/start-dfs.sh
./hadoop/sbin/start-yarn.sh
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstophadoop
#!/bin/bash
echo "Stopping Hadoop..."
cd ${MY_SW_ROOT}
. $MY_SW_SCRIPTS_DIR/sw_env.sh
./hadoop/sbin/stop-yarn.sh
./hadoop/sbin/stop-dfs.sh
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstartmysql
#!/bin/bash
echo "Starting mysqld..."
cd ${MY_SW_ROOT}/mysql
. $MY_SW_SCRIPTS_DIR/sw_env.sh
./bin/mysqld_safe --defaults-file=${MY_SQL_CONFIG_FILE} --log-error=${MY_SW_ROOT}/log/mysqld_safe.\$HOSTNAME.log &
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstopmysql
#!/bin/bash
echo "Stopping mysqld..."
cd ${MY_SW_ROOT}/mysql
. $MY_SW_SCRIPTS_DIR/sw_env.sh
./bin/mysqladmin --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_ADMIN} --password=${MY_SQL_ADMIN_PASSWD}  shutdown
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstarthbase
#!/bin/bash
echo "Starting HBase..."
cd ${MY_SW_ROOT}/hbase
. $MY_SW_SCRIPTS_DIR/sw_env.sh
bin/start-hbase.sh
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstophbase
#!/bin/bash
cd ${MY_SW_ROOT}/hbase
. $MY_SW_SCRIPTS_DIR/sw_env.sh
bin/stop-hbase.sh
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swstatus
#!/bin/bash
cd \${MY_SW_ROOT}
. \$MY_SW_SCRIPTS_DIR/sw_env.sh
JPS_OUTPUT=\`jps\`
SERVICES='HMaster NodeManager ResourceManager NameNode DataNode SecondaryNameNode'
for s in \$SERVICES; do
    if [[ ! \$JPS_OUTPUT =~ \$s ]]; then
        MISS_SERVICE="\$s \$MISS_SERVICE"
    fi
done

if [[ \$MISS_SERVICE != '' ]]; then
    echo "ERROR: Service \"\$MISS_SERVICE\" are not up!"
    exit 1
else
    echo "The local hadoop services are up!"
fi

NUM_MYSQLD_PROCS=\`ps -aef | grep \$USER | grep mysqld | grep -v grep | wc -l\`
if [[ \$NUM_MYSQLD_PROCS -ne 0 ]]; then
    echo "\$NUM_MYSQLD_PROCS mysqld processes are running"
    exit 0
else
    echo "ERROR: mysqld process is not running!"
    exit 1
fi
EOF

  #######################################################
  # scripts to start command line interpreters and tools
  #######################################################
  cat <<EOF >$MY_SW_SCRIPTS_DIR/swhadoop
#!/bin/bash
# command to run hadoop

. $MY_SW_SCRIPTS_DIR/sw_env.sh
${MY_SW_ROOT}/hadoop/bin/hadoop "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swyarn
#!/bin/bash
# command to run yarn

. $MY_SW_SCRIPTS_DIR/sw_env.sh
${MY_SW_ROOT}/hadoop/bin/yarn "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swhdfs
#!/bin/bash
# command to run hadoop

. $MY_SW_SCRIPTS_DIR/sw_env.sh
${MY_SW_ROOT}/hadoop/bin/hdfs "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swmysql
#!/bin/bash
# command to run mysql

. $MY_SW_SCRIPTS_DIR/sw_env.sh
${MY_SW_ROOT}/mysql/bin/mysql --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_USER} --password=${MY_SQL_USER_PASSWD} --database=${MY_SQL_METASTORE_DB} "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swmysqladmin
#!/bin/bash
# command to run mysqladmin as root user

. $MY_SW_SCRIPTS_DIR/sw_env.sh
${MY_SW_ROOT}/mysql/bin/mysqladmin --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_ADMIN} --password=${MY_SQL_ADMIN_PASSWD} "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swhive
#!/bin/bash
# command to run hive command line interpreter

# Pick up MySQL JDBC driver
. $MY_SW_SCRIPTS_DIR/sw_env.sh
export HADOOP_HOME=${MY_SW_ROOT}/hadoop

if [[ "$HBASE_DISTRO" = "APACHE" ]]; then
  cd ${MY_SW_ROOT}/hadoop/share/hadoop/yarn/lib
  rm jline*.jar
fi

${MY_SW_ROOT}/hive/bin/hive "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swhbase
#!/bin/bash
# command to run hbase shell

. $MY_SW_SCRIPTS_DIR/sw_env.sh
${MY_SW_ROOT}/hbase/bin/hbase shell "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swzkcli
#!/bin/bash
# command to run hbase zkcli

. $MY_SW_SCRIPTS_DIR/sw_env.sh
${MY_SW_ROOT}/hbase/bin/hbase zkcli "\$@"
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swuninstall_local_hadoop
#!/bin/bash
# uninstall local Hadoop instance.

. $MY_SW_SCRIPTS_DIR/swstopall
echo "Removing directory $MY_SW_ROOT"
echo "and all of its content."
echo "All Hadoop, HDFS, Hive, HBase content on this local instance will be lost."
echo -n "Is this ok? (y, (n)) "
read YN

if [ "\$YN" = "y" -o "\$YN" = "Y" ]; then
  rm -rf $MY_SW_ROOT
  # also remove the convenience scripts
  cds
  rm sw*
  echo "Removed $MY_SW_ROOT"
else
  echo "Exiting without removing anything..."
fi
EOF

  cat <<EOF >$MY_SW_SCRIPTS_DIR/swurls.html
<HTML>
<HEAD>
<title>URLs for local Hadoop Instance</title>
</HEAD>
<BODY>
<pre>
<a href="http://${MY_HOST_1}:${MY_HADOOP_NN_HTTP_PORT_NUM}">HDFS Admin</a>
<a href="http://${MY_HOST_1}:${MY_YARN_HTTP_PORT_NUM}">Yarn</a>
<a href="http://${MY_HOST_1}:${MY_HBASE_MASTER_INFO_PORT_NUM}">HBase Master</a>
<a href="http://${MY_HOST_1}:${MY_DCS_MASTER_INFO_PORT}">DCS Master</a>
<a href="http://${MY_HOST_1}:${MY_DRILL_INFO_PORT_NUM}">Drill Web Console (while shell is running)</a>
</pre>
</BODY>
</HTML>
EOF


  SYSTEM_DEFAULTS_TEXT_FILE=$TRAF_CONF/SQSystemDefaults.conf

  # alternative method for open source build, system defaults in a text file
  cat <<EOF >${SYSTEM_DEFAULTS_TEXT_FILE}.temp
#+-+-+ install_local_hadoop inserted this - please do not edit this section
# No default settings added by install_local_hadoop at this time
#-+-+- install_local_hadoop end of system-inserted values, please add your own custom values below
EOF

  # preserve any other text in the system defaults text file, remove old generated text
  if [ -r ${SYSTEM_DEFAULTS_TEXT_FILE} ]; then
    sed '/#+-+-+ install_local_hadoop /,/#-+-+- install_local_hadoop /d' <${SYSTEM_DEFAULTS_TEXT_FILE} >>${SYSTEM_DEFAULTS_TEXT_FILE}.temp
  fi
  # now overwrite the system defaults text file
  cat ${SYSTEM_DEFAULTS_TEXT_FILE}.temp >${SYSTEM_DEFAULTS_TEXT_FILE}

  # secure all shell scripts to be executable
  chmod +x $MY_SW_SCRIPTS_DIR/sw*

  echo
  echo "Checking permissions on home directory..."
  ls -ld ~
  ls -ld ~ | egrep 'drwx.-..-.' >/dev/null
  if [ $? -ne 0 ]; then
    echo '**** ERROR:'
    cat <<EOF
    Check permissions on your home directory. Authentication with
    public/private keys won't work if you allow write access to
    your home directory. You can fix this by doing something like
    the following:

    chmod 755 ~

    exiting, please correct and retry this script...
EOF
    exit
  fi

  # check password-less login via ssh
  check_ssh

fi
# end of general setup of sw directory

cd $MY_SW_ROOT
echo
if [ -d hadoop/bin ]; then
  echo "Hadoop files already exist, skipping Hadoop setup"
else
  echo "Setting up Hadoop..."
  #####################################################
echo "$MY_LOCAL_SW_DIST/${HADOOP_TAR}"
  if [ -f $MY_LOCAL_SW_DIST/${HADOOP_TAR} ]; then
    cp $MY_LOCAL_SW_DIST/${HADOOP_TAR} .
    echo "Copied Hadoop tar file from: $MY_LOCAL_SW_DIST/${HADOOP_TAR}" 
  else
    curl -O ${HADOOP_MIRROR_URL}/${HADOOP_TAR}
    echo "Downloaded Hadoop tar file: ${HADOOP_MIRROR_URL}/${HADOOP_TAR}"
  fi

  echo "Unpacking Hadoop tar file..."
  tar -xf ${HADOOP_TAR}
  rm -rf hadoop
  ln -s `dirname hadoop-*/lib` hadoop

  H_CORE_FILE=hadoop/etc/hadoop/core-site.xml
  H_HDFS_FILE=hadoop/etc/hadoop/hdfs-site.xml
  H_MAPRED_FILE=hadoop/etc/hadoop/mapred-site.xml
  H_Y_SITE_FILE=hadoop/etc/hadoop/yarn-site.xml
  H_ENV_FILE=hadoop/etc/hadoop/hadoop-env.sh
  H_Y_ENV_FILE=hadoop/etc/hadoop/yarn-env.sh

  echo "Updating files $H_CORE_FILE, $H_HDFS_FILE and $H_MAPRED_FILE ..."

  mv -f $H_CORE_FILE $H_CORE_FILE.orig
  cat <<EOF >$H_CORE_FILE
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
     <property>
         <name>fs.default.name</name>
         <value>hdfs://${MY_HOST_1}:${MY_HADOOP_HDFS_PORT_NUM}</value>
     </property>
     <property>
         <name>hadoop.tmp.dir</name>
         <value>${MY_HADOOP_DATA_DIR}/tmp</value>
     </property>
</configuration>
EOF

  mv -f $H_HDFS_FILE $H_HDFS_FILE.orig
  cat <<EOF >$H_HDFS_FILE
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
     <property>
         <name>dfs.replication</name>
         <value>1</value>
     </property>
     <property>
         <name>dfs.datanode.data.dir</name>
         <value>${MY_HADOOP_DATA_DIR}/dfs/data</value>
     </property>
     <property>
         <name>dfs.namenode.name.dir</name>
         <value>file://${MY_HADOOP_DATA_DIR}/dfs/name</value>
     </property>

     <property>
         <name>dfs.http.address</name>
         <value>${MY_HOST_1}:${MY_HADOOP_NN_HTTP_PORT_NUM}</value>
     </property>
     <property>
         <name>dfs.secondary.http.address</name>
         <value>${MY_HOST_1}:${MY_HADOOP_SECONDARY_NN_PORT_NUM}</value>
     </property>
     <property>
         <name>dfs.datanode.address</name>
         <value>${MY_HOST_1}:${MY_HADOOP_DN_PORT_NUM}</value>
     </property>
     <property>
         <name>dfs.datanode.http.address</name>
         <value>${MY_HOST_1}:${MY_HADOOP_DN_HTTP_PORT_NUM}</value>
     </property>
     <property>
         <name>dfs.datanode.ipc.address</name>
         <value>${MY_HOST_1}:${MY_HADOOP_DN_IPC_PORT_NUM}</value>
     </property>

     <property>
         <name>dfs.namenode.acls.enabled</name>
         <value>true</value>
     </property>
</configuration>
EOF

  if [ -r $H_MAPRED_FILE ]; then
    mv -f $H_MAPRED_FILE $H_MAPRED_FILE.orig
  fi
  cat <<EOF >$H_MAPRED_FILE
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
     <property>
         <name>mapreduce.framework.name</name>
         <value>local</value>
     </property>
     <property>
         <name>mapreduce.jobtracker.http.address</name>
         <value>${MY_HOST_1}:${MY_HADOOP_JOB_TRACKER_HTTP_PORT_NUM}</value>
     </property>
     <property>
         <name>mapreduce.tasktracker.http.address</name>
         <value>${MY_HOST_1}:${MY_HADOOP_TASK_TRACKER_PORT_NUM}</value>
     </property>
     <property>
         <name>mapreduce.shuffle.port</name>
         <value>${MY_HADOOP_SHUFFLE_PORT_NUM}</value>
     </property>
</configuration>
EOF

  mv -f $H_Y_SITE_FILE $H_Y_SITE_FILE.orig
  cat <<EOF >$H_Y_SITE_FILE
<?xml version="1.0"?>
<configuration>
  <property>
    <name>yarn.resourcemanager.address</name>
    <value>${MY_HOST_1}:${MY_YARN_RESMAN_PORT_NUM}</value>
  </property>
  <property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>${MY_HOST_1}:${MY_YARN_SCHED_PORT_NUM}</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address	</name>
    <value>${MY_HOST_1}:${MY_YARN_HTTP_PORT_NUM}</value>
  </property>
  <property>
    <name>yarn.resourcemanager.resource-tracker.address	</name>
    <value>${MY_HOST_1}:${MY_YARN_TRACKER_PORT_NUM}</value>
  </property>
  <property>
    <name>yarn.resourcemanager.admin.address	</name>
    <value>${MY_HOST_1}:${MY_YARN_ADMIN_PORT_NUM}</value>
  </property>
  <property>
    <name>yarn.nodemanager.localizer.address	</name>
    <value>${MY_HOST_1}:${MY_YARN_LOCALIZER_PORT_NUM}</value>
  </property>
  <property>
    <name>yarn.nodemanager.webapp.address</name>
    <value>${MY_HOST_1}:${MY_YARN_NM_PORT_NUM}</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
    <description>shuffle service that needs to be set for Map Reduce to run </description>
  </property>
</configuration>
EOF

  echo "Appending local configuration to $H_ENV_FILE..."
  cat <<EOF >>$H_ENV_FILE

# Trafodion-local configuration
# Make sure JAVA_HOME is set
export JAVA_HOME=${JAVA_HOME}
# Use a local PID dir to avoid conflicts with other Hadoop instances
export HADOOP_PID_DIR=${MY_SW_ROOT}/log
EOF

  echo "Appending local configuration to $H_Y_ENV_FILE..."
  cat <<EOF >>$H_Y_ENV_FILE

# Trafodion-local configuration
# Make sure JAVA_HOME is set
export JAVA_HOME=${JAVA_HOME}
export HADOOP_CONF_DIR=${YARN_HOME}/etc/hadoop
export HADOOP_COMMON_HOME=${YARN_HOME}
export HADOOP_HDFS_HOME=${YARN_HOME}
# Use a local PID dir to avoid conflicts with other Hadoop instances
export HADOOP_PID_DIR=${MY_SW_ROOT}/log
EOF

  #####################################################
  echo "Initializing and starting Hadoop..." | tee ${MY_LOG_FILE}

  cd $MY_SW_ROOT/hadoop
  . $MY_SW_SCRIPTS_DIR/sw_env.sh

  bin/hdfs namenode -format         >>${MY_LOG_FILE} 2>&1
  $MY_SW_SCRIPTS_DIR/swstophadoop   >>${MY_LOG_FILE} 2>&1
  $MY_SW_SCRIPTS_DIR/swstarthadoop  >>${MY_LOG_FILE} 2>&1

  echo "Creating HDFS directories" 2>&1 | tee -a ${MY_LOG_FILE}
  bin/hdfs dfs -mkdir /tmp                           >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -mkdir /user                          >>${MY_LOG_FILE} 2>&1  
  bin/hdfs dfs -mkdir /user/trafodion                >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -mkdir /user/trafodion/$USER          >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -mkdir /user/trafodion/hive           >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -mkdir /user/trafodion/bulkload       >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -mkdir /user/trafodion/hive/warehouse >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -mkdir /user/trafodion/hive           >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -chmod g+w /tmp                       >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -chmod g+w /user/trafodion/hive/warehouse   >>${MY_LOG_FILE} 2>&1
  bin/hdfs dfs -chmod g+w /user/trafodion/bulkload   >>${MY_LOG_FILE} 2>&1
  bin/hadoop fs -ls -R /                 2>&1 | tee -a ${MY_LOG_FILE}
  echo "Done: Creating HDFS directories" 2>&1 | tee -a ${MY_LOG_FILE}

fi
# end of Hadoop (MapReduce + HDFS) setup

cd $MY_SW_ROOT

if [ -d mysql/bin ]; then
  echo "MySQL files already exist, skipping MySQL setup"
else
  #####################################################
  echo "Setting up MySQL..."
echo "$MY_LOCAL_SW_DIST/${MYSQL_TAR}"
  if [ -f $MY_LOCAL_SW_DIST/${MYSQL_TAR} ]; then
    cp $MY_LOCAL_SW_DIST/${MYSQL_TAR} .
    echo "Copied MySql tar file from: $MY_LOCAL_SW_DIST/${MYSQL_TAR}"
  else
    curl ${MYSQL_MIRROR_URL}/${MYSQL_TAR} -o ${MYSQL_TAR}
    echo "Downloaded MySql tar file: ${MYSQL_MIRROR_URL}/${MYSQL_TAR}"
  fi

  echo "Unpacking MySQL tar file ${MYSQL_TAR} ..."
  tar -xf ${MYSQL_TAR}
  echo "Creating symbolic link to latest MySQL distribution..."
  rm -rf mysql
  ln -s `dirname mysql-*-linux-*/bin` mysql

  # set up MySQL configuration file

  if [ -f ${MY_SQL_CONFIG_FILE} ]; then
    echo "Using existing MySQL config file ${MY_SQL_CONFIG_FILE}"
  else
    echo "Setting up MySQL configuration file ${MY_SQL_CONFIG_FILE} ..."
    mkdir ${MY_SQL_DATA_DIR}
    cat <<EOF >${MY_SQL_CONFIG_FILE}
[client-server]
# Uncomment these if you want to use a nonstandard connection to MySQL
port=${MY_SQL_PORT_NUM}
socket=/tmp/mysql_${MY_SQL_PORT_NUM}.sock

# This will be passed to all MySQL clients
[client]
port=${MY_SQL_PORT_NUM}
socket=/tmp/mysql_${MY_SQL_PORT_NUM}.sock

# The MySQL server
[mysqld]
# port to use (default is 3306)
port=${MY_SQL_PORT_NUM}
socket=/tmp/mysql_${MY_SQL_PORT_NUM}.sock
# Directory where you want to put your data
datadir=${MY_SQL_DATA_DIR}
# File that contains the pid of the running mysqld
pid_file=${MY_SW_ROOT}/log/mysqld.pid
# Directory for error messages
lc-messages-dir=${MY_SW_ROOT}/mysql/share
# Create a file where the InnoDB/XtraDB engine stores it's data
# innodb_data_file_path = ibdata1:20M;ibdata2:40M:autoextend
# innodb_file_per_table

# Enable logging by default to help find problems
general-log=1
general-log-file=${MY_SW_ROOT}/log/mysql-general.${HOSTNAME}.log

EOF

  fi

  cd mysql
  echo "Running MySQL installation script..." | tee -a ${MY_LOG_FILE}
  ./scripts/mysql_install_db --defaults-file=${MY_SQL_CONFIG_FILE} >>${MY_LOG_FILE} 2>&1
  if [ $? -ne 0 ]; then
    echo "Problem installing MySQL, see, file ${MY_LOG_FILE}" | tee -a ${MY_LOG_FILE}
  fi

  # start mysql
  echo "Starting mysqld..." | tee -a ${MY_LOG_FILE}
  $MY_SW_SCRIPTS_DIR/swstartmysql >>${MY_LOG_FILE} 2>&1
  if [ $? -ne 0 ]; then
    echo "Problem starting MySQL, see, file ${MY_LOG_FILE}" | tee -a ${MY_LOG_FILE}
  fi
fi
# end of MySQL setup

cd $MY_SW_ROOT

if [ -f mysql-connector-java-*/mysql-connector-java-*.jar ]; then
  echo "MySQL JDBC driver file already exists, skipping JDBC setup"
else
  #####################################################
  echo "Setting up MySQL JDBC driver..."
echo "$MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR}"
  if [ -f $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR} ]; then
    cp $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR} .
    echo "Copied MySql JDBC tar file from: $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR}"
  else
    curl -O ${MYSQL_JDBC_URL}/${MYSQL_JDBC_TAR}
    echo "Downloaded MySql JDBC tar file: ${MYSQL_JDBC_URL}/${MYSQL_JDBC_TAR}"
  fi

  echo "Unpacking MySQL JDBC tar file ${MYSQL_JDBC_TAR} ..."
  tar -xf ${MYSQL_JDBC_TAR}
  ln -s `dirname mysql-connector-java-*/mysql-connector-java-*.jar` mysql-connector-java
  # copy the MySQL JDBC driver to the udr directory, so UDRs can use it
  mkdir -p ${TRAF_VAR}/udr/external_libs
  cp ${MY_SW_ROOT}/mysql-connector-java/mysql-connector-java*.jar ${TRAF_VAR}/udr/external_libs

fi
# end of MySQL JDBC setup

cd $MY_SW_ROOT

if [ -d hive/bin ]; then
  echo "Hive files already exist, skipping Hive setup"
else
  #####################################################
  echo "Setting up Hive..."
echo "$MY_LOCAL_SW_DIST/${HIVE_TAR}"
  if [ -f $MY_LOCAL_SW_DIST/${HIVE_TAR} ]; then
    cp $MY_LOCAL_SW_DIST/${HIVE_TAR} .
    echo "Copied Hive tar file from: $MY_LOCAL_SW_DIST/${HIVE_TAR}"
  else
    curl -O ${HIVE_MIRROR_URL}/${HIVE_TAR}
    echo "Downloaded Hive tar file: ${HIVE_MIRROR_URL}/${HIVE_TAR}"
  fi

  echo "Unpacking Hive tar file ${HIVE_TAR} ..."
  tar -xf ${HIVE_TAR}
  echo "Creating symbolic link to latest Hive distribution..."
  rm -rf hive
#  ln -s `dirname hive-*/bin` hive
  ln -s ${HIVE_PREFIX} hive

  HIVE_CONFIG_FILE=hive/conf/hive-site.xml
  HIVE_LOG_CONFIG_FILE=hive/conf/hive-log4j.properties

  echo "Updating file $HIVE_CONFIG_FILE ..."

  if [ -r $HIVE_CONFIG_FILE ]; then
    mv -f $HIVE_CONFIG_FILE $HIVE_CONFIG_FILE.orig
  fi
  cat <<EOF >$HIVE_CONFIG_FILE
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
  <property>
     <name>hive.exec.scratchdir</name>
     <value>${MY_HIVE_DATA_DIR}</value>
  </property>

  <property>
     <name>hive.metastore.local</name>
     <value>true</value>
  </property>

  <!-- Use MySQL as metastore -->

  <property>
     <name>javax.jdo.option.ConnectionURL</name>
     <value>jdbc:mysql://${MY_HOST_1}:${MY_SQL_PORT_NUM}/${MY_SQL_METASTORE_DB}?createDatabaseIfNotExist=true</value>
     <description>JDBC connect string for a JDBC metastore (don't include white space)</description>
  </property>

  <property>
     <name>javax.jdo.option.ConnectionDriverName</name>
     <value>com.mysql.jdbc.Driver</value>
     <description>Driver class name for a JDBC metastore</description>
  </property>

  <property>
     <name>javax.jdo.option.ConnectionUserName</name>
     <value>${MY_SQL_USER}</value>
  </property>

  <property>
     <name>javax.jdo.option.ConnectionPassword</name>
     <value>${MY_SQL_USER_PASSWD}</value>
  </property>

  <!-- end of MySQL metastore parameters -->

  <!-- Alternatively, use this to set up Apache Derby as metadata store

  <property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:derby:;databaseName=${MY_DERBY_DATA_DIR}/metastore_db;create=true</value>
    <description>JDBC connect string for a JDBC metastore</description>
  </property>

  end of alternative Derby configuration -->

  <!-- other useful Hive configuration settings -->
  <property>
    <name>hive.cli.print.header</name>
    <value>true</value>
    <description>Whether to print the names of the columns in query output.</description>
  </property>
</configuration>
EOF

  cat <<EOF >${HIVE_LOG_CONFIG_FILE}
hive.root.logger=WARN,DRFA
hive.log.dir=${MY_SW_ROOT}/log/hive
hive.log.file=hive.log

EOF

  echo "Copying MySQL JDBC driver to ${MY_SW_ROOT}/hive/lib"
  cp -p ${MY_SW_ROOT}/mysql-connector-java/*.jar ${MY_SW_ROOT}/hive/lib

  echo "Sleeping 10 sec to wait for MySQL to start..."
  sleep 10

  mysql/bin/mysqladmin --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_ADMIN} password ${MY_SQL_ADMIN_PASSWD} 2>&1 | tee -a ${MY_LOG_FILE}
  mysql/bin/mysqladmin --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_ADMIN} --password=${MY_SQL_ADMIN_PASSWD} -h ${MY_HOST_1} password ${MY_SQL_ADMIN_PASSWD} 2>&1 | tee -a ${MY_LOG_FILE}

  if [ ${MY_SQL_ADMIN} != ${MY_SQL_USER} ]; then
    echo "Creating MySQL user ${MY_SQL_USER}"
    ${MY_SW_ROOT}/mysql/bin/mysql --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_ADMIN} --password=${MY_SQL_ADMIN_PASSWD} <<EOF 2>&1 | tee -a ${MY_LOG_FILE}
CREATE USER '${MY_SQL_USER}'@'%' IDENTIFIED BY '${MY_SQL_USER_PASSWD}';
EOF
    mysql/bin/mysqladmin --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_USER} password ${MY_SQL_USER_PASSWD} 2>&1 | tee -a ${MY_LOG_FILE}
    mysql/bin/mysqladmin --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_USER} --password=${MY_SQL_USER_PASSWD} -h ${MY_HOST_1} password ${MY_SQL_USER_PASSWD} 2>&1 | tee -a ${MY_LOG_FILE}
  fi

  echo "Creating Hive database and metastore in MySQL..."
  ${MY_SW_ROOT}/mysql/bin/mysql --defaults-file=${MY_SQL_CONFIG_FILE} --user=${MY_SQL_USER} --password=${MY_SQL_USER_PASSWD} <<EOF 2>&1 | tee -a ${MY_LOG_FILE}
create database ${MY_SQL_METASTORE_DB};
use ${MY_SQL_METASTORE_DB};
SOURCE ${MY_SW_ROOT}/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;
EOF

fi
# end of Hive setup

cd $MY_SW_ROOT

if [ -d hbase/bin ]; then
  echo "HBase files already exist, skipping HBase setup"
else
  #####################################################
  echo "Setting up HBase..."
echo "$MY_LOCAL_SW_DIST/${HBASE_TAR}"
  if [ -f $MY_LOCAL_SW_DIST/${HBASE_TAR} ]; then
    cp $MY_LOCAL_SW_DIST/${HBASE_TAR} .
    echo "Copied HBase tar file from: $MY_LOCAL_SW_DIST/${HBASE_TAR}"
  else
    curl -O ${HBASE_MIRROR_URL}/${HBASE_TAR}
    echo "Downloaded HBase tar file: ${HBASE_MIRROR_URL}/${HBASE_TAR}"
  fi

  echo "Unpacking HBase tar file ${HBASE_TAR} ..."
  tar -xf ${HBASE_TAR}
  echo "Creating symbolic link to latest HBase distribution..."
  rm -rf hbase
  ln -s `dirname hbase-*/bin` hbase

  HBASE_CONFIG_FILE=hbase/conf/hbase-site.xml
  HBASE_ENV_FILE=hbase/conf/hbase-env.sh

  # For HBase_Trx

  # Do not build if the TRX jar already exists
  if [ ! -e ${TRAF_HOME}/export/lib/${HBASE_TRX_JAR} ]; then
     # Build HBase TRX
     echo "Building HBase TRX"
     cd $TRAF_HOME ; make genverhdr 2>&1 | tee -a ${MY_LOG_FILE}
     cd $TRAF_HOME/src/seatrans/hbase-trx
     make clean 2>&1 | tee -a ${MY_LOG_FILE}
     make 2>&1 | tee -a ${MY_LOG_FILE}
  fi

  cd $MY_SW_ROOT

  # Setup HBase TRX JAR and utility JAR in HBase' CLASSPATH
  echo "export HBASE_CLASSPATH=${TRAF_HOME}/export/lib/\${HBASE_TRX_JAR}:${TRAF_HOME}/export/lib/\${UTIL_JAR}" >> ${HBASE_ENV_FILE}

  echo "Updating file $HBASE_CONFIG_FILE ..."

  if [ -r $HBASE_CONFIG_FILE ]; then
    mv -f $HBASE_CONFIG_FILE $HBASE_CONFIG_FILE.orig
  fi

  if [[ "$HBASE_DISTRO" =~ "APACHE" ]]; then
    MASTER_VISIBILITY_COPROC="<property>
    <name>hbase.coprocessor.master.classes</name>
    <value>org.apache.hadoop.hbase.security.access.AccessController,
           org.apache.hadoop.hbase.security.visibility.VisibilityController
    </value>
    </property>"
    VISIBILITY_COPROC=", 
      org.apache.hadoop.hbase.security.visibility.VisibilityController"
  fi

  if [[ "$HBASE_DISTRO" != "APACHE1.2" && "$HBASE_DISTRO" != "CDH5.7" ]]; then 
    HREGION_IMPL="<property>
    <name>hbase.hregion.impl</name>
    <value>org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion</value>
    </property>"
  fi

  cat <<EOF >$HBASE_CONFIG_FILE
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
  <property>
    <name>hbase.rootdir</name>
    <value>hdfs://${MY_HOST_1}:${MY_HADOOP_HDFS_PORT_NUM}/hbase</value>
  </property>
  <property>
   <name>hbase.zookeeper.quorum</name>
   <value>${MY_HOST_1}</value>
 </property>
 <property>
    <name>hbase.zookeeper.property.dataDir</name>
    <value>${MY_ZOOKEEPER_DATA_DIR}</value>
  </property>
  <property>
    <name>hbase.master.port</name>
    <value>${MY_HBASE_MASTER_PORT_NUM}</value>
  </property>
  <property>
    <name>hbase.master.info.port</name>
    <value>${MY_HBASE_MASTER_INFO_PORT_NUM}</value>
  </property>
  <property>
    <name>hbase.regionserver.port</name>
    <value>${MY_HBASE_REGIONSERVER_PORT_NUM}</value>
  </property>
  <property>
    <name>hbase.regionserver.info.port</name>
    <value>${MY_HBASE_REGIONSERVER_INFO_PORT_NUM}</value>
  </property>
  <property>
    <name>hbase.zookeeper.peerport</name>
    <value>${MY_HBASE_ZOOKEEPER_PEERPORT_NUM}</value>
  </property>
  <property>
    <name>hbase.zookeeper.leaderport</name>
    <value>${MY_HBASE_ZOOKEEPER_LEADERPORT_NUM}</value>
  </property>
  <property>
    <name>hbase.zookeeper.property.clientPort</name>
    <value>${MY_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT_NUM}</value>
  </property>
  <property>
    <name>hbase.rest.port</name>
    <value>${MY_HBASE_REST_PORT_NUM}</value>
  </property>
  <property>
    <name>hbase.client.scanner.caching</name>
	<value>100</value>
  </property>
  <property>
    <name>hbase.client.scanner.timeout.period</name>
	<value>60000</value>
  </property>
  <property>
     <name>hbase.bulkload.staging.dir</name>
     <value>hdfs://${MY_HOST_1}:${MY_HADOOP_HDFS_PORT_NUM}/hbase-staging</value>
  </property>
 <property>
    <name>hbase.snapshot.enabled</name>
    <value>true</value>
  </property>
   ${HREGION_IMPL}
   ${MASTER_VISIBILITY_COPROC}
</configuration>
EOF

  echo "Starting HBase..." | tee -a ${MY_LOG_FILE}
  $MY_SW_SCRIPTS_DIR/swstarthbase >>${MY_LOG_FILE} 2>&1
  if [ $? -ne 0 ]; then
    echo "Problem starting HBase, see, file ${MY_LOG_FILE}" | tee -a ${MY_LOG_FILE}
  fi

fi
# end of HBase setup

if [[ ! -z $FAST_LOCAL_HADOOP ]]; then
    echo "FAST_LOCAL_HADOOP is set. Exiting..."
    exit 0
fi

cd $MY_SW_ROOT

if [ $INSTALL_TPCDS -eq 1 ]; then
  if [ -d tpcds/tools ]; then
    echo "TPC-DS files already exist, skipping TPC-DS setup"
  else
    install_hadoop_regr_test_env \
        --unpackDir=$MY_SW_ROOT/tpcds \
        --dataDir=${MY_DATA_DIR}/tpcds \
        --logFile=$MY_LOG_FILE \
        --hdfsCmd=${MY_SW_SCRIPTS_DIR}/swhdfs \
        --hiveCmd=${MY_SW_SCRIPTS_DIR}/swhive
    if [ $? -ne 0 ]; then
      echo "Error installing TPC-DS and ORC files, exiting..."
      exit 1
    fi
  fi
fi
# end of TPC-DS setup

cd $MY_SW_ROOT

if [[ -n "$VERBOSE" ]]; then

  if [ ! -r $YARN_HOME/lib/native/libhdfs.so ]; then
    echo "------------------------------------------------------------"
    echo "-- WARNING: libhdfs.so is not present on this system. Please"
    echo "--          build it, otherwise Trafodion will not compile."
    echo "------------------------------------------------------------"
  fi
fi

echo
echo "Installed directory size and name = $(du -sh $MY_SW_ROOT)" | tee -a ${MY_LOG_FILE}
echo
echo "Setup is complete. You can use the convenience scripts starting with sw... located in $MY_SW_SCRIPTS_DIR."