install/traf_tools_setup.sh

#!/bin/bash
#
# @@@ START COPYRIGHT @@@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# @@@ END COPYRIGHT @@@
#

# -----------------------------------------------------------------------------
# script: traf_tools_setup
#
# Helper script that downloads and installs dependent tools required by 
# Apache Trafodion to build from a source distribution
#
# Tools installed:
#
# MPICH: Implementation of the Message Passing Interface (MPI) standard.  For use 
#   in Trafodion, MPICH must be built to force sockets to be used in both 
#   internode and intranode message passing. 
# Bison: General-purpose parser generator.
# Udis86: Minimalistic disassembler library (libudis86) for the x86 class of 
#   instruction set architectures.
# LLVM: Collection of modular and reusable compiler and toolchain technologies.
# ICU: C/C++ and Java libraries providing Unicode and Globalization support for 
#   software applications.
# Zookeeper: Coordination service for distributed applications.  It exposes 
#   common services such as naming, configuration management, synchronization, 
#   and group services.
# Thrift: Communications and data serialization tool
# Maven: Build tool that is only installed if compatible version does not exist
# log4cxx: standard logging framework for C++
# hadoop: shared libraries for libhadoop, libhdfs, and hdfs header file
#
# Script can be modified to meet the needs of your environment
# May need root or SUDO access to install tools in desired location
# ----------------------------------------------------------------------------- 

##############################################################################
# Environment variables - optional
#
# MY_LOCAL_SW_DIST - shared location on local network for tar balls for build, etc.
# - udis
# - llvm
# - mpich
# - bison
# - icu
# - zookeeper
# - thrift
# - apache-maven
# - protobuf
# - apache-log4cxx
# - hadoop
# also set http_proxy and ftp_proxy if necessary, to download
# files from repositories on the Internet
#
##############################################################################
MY_LOCAL_SW_DIST=${MY_LOCAL_SW_DIST-""}

function Usage {
   echo
   echo "Usage: $0 -d <downloaddir> -i <installdir>"
   echo
   echo " -d <downloaddir> - location of download directory"
   echo " -i <installdir>  - location of install directory"
   echo
   echo " -h - help"
   echo
   echo "example: traf_tools_setup.sh -d /home/userx/download -i /home/userx/tools"
   echo
}

# -----------------------------------------------------------------------------
# function: downloadSource - downloads and un-tars the requested file
#    $1 - tar file URL to download
#    $2 - directory where source is untarred (leave empty to skip untar)
#
# -----------------------------------------------------------------------------
function downloadSource
{
  URL="$1"
  SRCDIR="$2"
  TARFILE="${URL##*/}"

  if [ "x${MY_LOCAL_SW_DIST}" != "x" ]; then
      echo "INFO:   copy file ${TARFILE} from [${MY_LOCAL_SW_DIST}/${TARFILE}]"
      if [ -f ${MY_LOCAL_SW_DIST}/${TARFILE} ]; then
	  cp ${MY_LOCAL_SW_DIST}/${TARFILE} ${BASEDIR}/${TARFILE}
      fi
  fi

  if [ ! -e $BASEDIR/$TARFILE ]; then
    wget $URL  >>$LOGFILE 2>&1
    echo "INFO:   downloaded tar file: $TARFILE " | tee -a $LOGFILE
  else
    echo "INFO:   tar file already downloaded, step skipped" | tee -a $LOGFILE
  fi

  if [ ! -e $BASEDIR/$SRCDIR ]; then
    cd $BASEDIR
    tar -xzf $BASEDIR/$TARFILE
  else
    echo "INFO:   source tree already exists" | tee -a $LOGFILE
  fi
}

LOGDIR=$(pwd)
LOGFILE=$LOGDIR/traf_tools_setup.log
BLDLOG=$LOGDIR/traf_tools_bld

# -----------------------------------------------------------------------------
# execute build functions in background
# $1 - short name of component (single word, also used in logfile name)
# $2 - first make target (usually empty string for the default target)
# $3 - second make target
# $4 - dir/file to check for success
# -----------------------------------------------------------------------------
function bkgBuild
{
  LOG=${BLDLOG}-${1}.log
  echo "INFO:   Starting background make and install for $1" | tee -a $LOGFILE

 (make $2 > $LOG 2>&1
  echo "INFO:   make completed" > $LOG
  make $3 > $LOG 2>&1
  if [[ ! -e $4 ]]; then
    echo "ERROR:  failed to install $1" | tee -a $LOGFILE
    echo "  see details in $LOG" | tee -a $LOGFILE
    exit 2;
  fi
  echo "INFO:   $1 install complete at $(date), files placed in $TOOLSDIR" | tee -a $LOGFILE
 ) &
}

# main code

TOOLSDIR=
BASEDIR=
rm $LOGFILE 2>/dev/null
echo
echo "INFO: Starting tools build on $(date)" | tee -a $LOGFILE

    while getopts "d:i:hv" arg
      do
      case $arg in
          d)
              BASEDIR=${OPTARG};
              ;;
          h)
              Usage;
              exit 1;
              ;;
          i)
              TOOLSDIR=${OPTARG};
              ;;
          *)
              Usage;
              exit 1;
              ;;
      esac
    done


if [ "$BASEDIR" == "" ]; then
  echo
  echo "ERROR: download directory (-d) is not specified"
  Usage;
  exit 1;
fi
# handle relative path
if [[ ! $BASEDIR =~ ^/ ]]
then
  BASEDIR=$(pwd)/$BASEDIR
fi

if [ ! -d "$BASEDIR" ]; then
  echo
  echo "ERROR: download directory ($BASEDIR) does not exist"
  echo
  exit 1;
fi
 
if [ "$TOOLSDIR" == "" ]; then
  echo
  echo "ERROR: install directory (-t)  is not specified"
  Usage;
  exit 1;
fi
# handle relative path
if [[ ! $TOOLSDIR =~ ^/ ]]
then
  TOOLSDIR=$(pwd)/$TOOLSDIR
fi

if [ ! -d "$TOOLSDIR" ]; then                                                    
  read -p "Tools install directory $TOOLSDIR does not exist, do you want to to create it? y/n : " CREATEDIR
  if [ "$CREATEDIR" == "y" ]; then
     mkdir $TOOLSDIR
     if [ ! -d "$TOOLSDIR" ]; then
       echo
       echo "ERROR: unable to create directory $TOOLSDIR"
       echo
       exit 1
     else
       echo "INFO: Created directory $TOOLSDIR" | tee -a $LOGFILE
     fi
  else
    echo
    echo "ERROR: install directory ($TOOLSDIR) does not exist"
    echo
    exit 1;
  fi
fi

echo "INFO: Tar download location: $BASEDIR" | tee -a $LOGFILE
echo "INFO: Tool install directory location: $TOOLSDIR" | tee -a $LOGFILE
echo "INFO: LogFile location: $LOGFILE" | tee -a $LOGFILE
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
# install udis
cd $BASEDIR
echo
echo "INFO: Installing UDIS on $(date)" | tee -a $LOGFILE
if [ -d $TOOLSDIR/udis86-1.7.2/bin ]; then
  echo "INFO: UDIS is already installed, skipping to next tool" | tee -a $LOGFILE
else
  downloadSource http://sourceforge.net/projects/udis86/files/udis86/1.7/udis86-1.7.2.tar.gz udis86-1.7.2
  cd udis86-1.7.2
  ./configure --prefix=$TOOLSDIR/udis86-1.7.2 --enable-shared >>$LOGFILE 2>&1
  echo "INFO:   configure complete" | tee -a $LOGFILE
  bkgBuild "udis86" "" "install" "$TOOLSDIR/udis86-1.7.2/bin"
fi
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
#install LLVM
cd $BASEDIR
echo
echo "INFO: Installing LLVM on $(date)" | tee -a $LOGFILE
if [ -d $TOOLSDIR/dest-llvm-3.2/release/bin -a -d $TOOLSDIR/dest-llvm-3.2/debug/bin ]; then
  echo "INFO: LLVM is already installed, skipping to next step" | tee -a $LOGFILE
else
  downloadSource http://llvm.org/releases/3.2/llvm-3.2.src.tar.gz llvm-3.2.src
  
  # Depends on UDIS, so make sure it is done
  wait

  export MY_UDIS_INSTALL_DIR=$TOOLSDIR/udis86-1.7.2
  export MY_LLVM_INSTALL_DIR=$TOOLSDIR/dest-llvm-3.2/
  export MY_LLVM_SRC_DIR=$BASEDIR/llvm-3.2.src
  export MY_LLVM_OBJ_DIR=$BASEDIR/llvm-3.2.obj/
  export LD_LIBRARY_PATH=$MY_UDIS_INSTALL_DIR/lib:$LD_LIBRARY_PATH
  export C_INCLUDE_PATH=$MY_UDIS_INSTALL_DIR/include
  export CPATH=$MY_UDIS_INSTALL_DIR/include

  # Build release version
  if [ ! -d $TOOLSDIR/dest-llvm-3.2/release/bin ]; then 
    mkdir -p $MY_LLVM_OBJ_DIR/release
    cd $MY_LLVM_OBJ_DIR/release

    $MY_LLVM_SRC_DIR/configure --prefix=$MY_LLVM_INSTALL_DIR/release \
       --enable-shared --enable-targets=x86,x86_64,cpp \
       --with-udis86=$MY_UDIS_INSTALL_DIR/lib \
       CFLAGS=-fgnu89-inline >>$LOGFILE 2>&1
    echo "INFO:   release configure complete" | tee -a $LOGFILE

    bkgBuild "llvm-release" "libs-only" "install-libs" "$TOOLSDIR/dest-llvm-3.2/release/bin"
  fi

  # Build debug version
  if [ ! -d $TOOLSDIR/dest-llvm-3.2/debug/bin ]; then
    mkdir -p $MY_LLVM_OBJ_DIR/debug
    cd $MY_LLVM_OBJ_DIR/debug

    $MY_LLVM_SRC_DIR/configure --prefix=$MY_LLVM_INSTALL_DIR/debug \
       --enable-optimized --enable-jit \
       --enable-debug-runtime --enable-debug-symbols \
       --enable-shared --enable-targets=x86,x86_64,cpp \
       --with-udis86=$MY_UDIS_INSTALL_DIR/lib \
       CFLAGS=-fgnu89-inline >>$LOGFILE 2>&1
    echo "INFO:   debug configure complete" | tee -a $LOGFILE
    bkgBuild "llvm-debug" "libs-only" "install-libs" "$TOOLSDIR/dest-llvm-3.2/debug/bin"
  fi
fi
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
# install mpi
cd $BASEDIR
echo
echo "INFO: Installing MPI on $(date)" | tee -a $LOGFILE
if [ -d $TOOLSDIR/dest-mpich-3.0.4/bin ]; then
  echo "INFO: MPI is already installed, skipping to next tool" | tee -a $LOGFILE
else	
  downloadSource http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz mpich-3.0.4 
  cd mpich-3.0.4
  ./configure --prefix=$TOOLSDIR/dest-mpich-3.0.4 --with-device=ch3:sock --disable-f77 --disable-fc >>$LOGFILE 2>&1
  echo "INFO:   configure complete" | tee -a $LOGFILE
  bkgBuild "MPI" "" "install" "$TOOLSDIR/dest-mpich-3.0.4/bin"
fi
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
# install bison
cd $BASEDIR
echo
echo "INFO: Installing Bison on $(date)" | tee -a $LOGFILE
if [ -d $TOOLSDIR/bison_3_linux/bin ]; then
  echo "INFO: Bison is already installed, skipping to next tool" | tee -a $LOGFILE
else	
  downloadSource http://ftp.gnu.org/gnu/bison/bison-3.0.tar.gz bison-3.0
  cd bison-3.0
  ./configure --prefix=$TOOLSDIR/bison_3_linux >>$LOGFILE 2>&1
  echo "INFO:   configure complete" | tee -a $LOGFILE
  bkgBuild "bison" "" "install" "$TOOLSDIR/bison_3_linux/bin"
fi
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
# Build ICU
cd $BASEDIR
echo
echo "INFO: Installing ICU on $(date)" | tee -a $LOGFILE
if [ -d $TOOLSDIR/icu4c_4.4/linux64/bin ]; then
  echo "INFO: ICU is already installed, skipping to next tool" | tee -a $LOGFILE
else
  downloadSource http://download.icu-project.org/files/icu4c/4.4/icu4c-4_4-src.tgz icu4c-4_4-src
  cd icu/source
  ./configure --prefix=$TOOLSDIR/icu4c_4.4/linux64 >>$LOGFILE 2>&1
  echo "INFO:   configure complete" | tee -a $LOGFILE
  bkgBuild "ICU" "" "install" "$TOOLSDIR/icu4c_4.4/linux64/bin"
fi
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
# make zookeeper
cd $BASEDIR
echo
echo "INFO: Installing ZooKeeper on $(date)" | tee -a $LOGFILE
if [ -d $TOOLSDIR/zookeeper-3.4.5/bin ]; then
  echo "INFO: ZooKeeper is already installed, skipping to next tool" | tee -a $LOGFILE
else
  downloadSource https://archive.apache.org/dist/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz zookeeper-3.4.5
  cd zookeeper-3.4.5/src/c
  ./configure --prefix=$TOOLSDIR/zookeeper-3.4.5 >>$LOGFILE 2>&1
  echo "INFO:   configure complete" | tee -a $LOGFILE
  bkgBuild "zookeeper" "" "install" "$TOOLSDIR/zookeeper-3.4.5/bin"
fi
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
# make thrift
cd $BASEDIR
echo
echo "INFO: Installing Thrift on $(date)" | tee -a $LOGFILE
if [ -d $TOOLSDIR/thrift-0.9.0/bin ]; then
  echo "INFO: Thrift is already installed, skipping to next tool" | tee -a $LOGFILE
else
  downloadSource http://archive.apache.org/dist/thrift/0.9.0/thrift-0.9.0.tar.gz thrift-0.9.0
  cd thrift-0.9.0
  ./configure --prefix=$TOOLSDIR/thrift-0.9.0 -without-qt >>$LOGFILE 2>&1
  echo "INFO:   configure complete" | tee -a $LOGFILE
  bkgBuild "thrift" "" "install" "$TOOLSDIR/thrift-0.9.0/bin"
fi
echo " *********************************************************** " | tee -a $LOGFILE


# -----------------------------------------------------------------------------
# install maven, if version 3 or greater not available already
MAVEN_VERSION=`mvn --version 2>/dev/null | grep 'Apache Maven' | cut -f 3 -d ' '`
if [[ ! "$MAVEN_VERSION" =~ "3." ]]; then
  cd $BASEDIR
  echo
  echo "INFO: Installing Maven on $(date)" | tee -a $LOGFILE
  if [ -d $TOOLSDIR/apache-maven-3.3.3/bin ]; then
    echo "INFO: Maven is already installed, skipping to next tool" | tee -a $LOGFILE
  else
    # there is no install, simply extract the files into $TOOLSDIR
    wget http://archive.apache.org/dist/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz >>$LOGFILE 2>&1
    cd $TOOLSDIR
    tar -xzf $BASEDIR/apache-maven-3.3.3-bin.tar.gz >>$LOGFILE 2>&1
    echo "INFO:   downloaded tar file:  apache-maven-3.3.3-bin.tar.gz" | tee -a $LOGFILE
  fi
    echo "INFO: Maven installation complete" | tee -a $LOGFILE
  echo " *********************************************************** " | tee -a $LOGFILE
else
  echo "INFO:  Maven is already installed, skipping to next tool" | tee -a $LOGFILE
fi

# -----------------------------------------------------------------------------
# install protobuf, if version 2.5 not available already
PROTOC_VERSION=`protoc --version 2>/dev/null | grep 'libprotoc' | cut -f 2 -d ' '`
if [[ ! "$PROTOC_VERSION" =~ "^2.5" ]]; then
  cd $BASEDIR
  echo
  echo "INFO: Installing Protobuf on $(date)" | tee -a $LOGFILE
  if [ -d $TOOLSDIR/protobuf-2.5.0/bin ]; then
    echo "INFO: Protobuf is already installed, skipping to next tool" | tee -a $LOGFILE
  else
    downloadSource https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz protobuf-2.5.0
    cd protobuf-2.5.0
    ./configure --prefix=$TOOLSDIR/protobuf-2.5.0 >>$LOGFILE 2>&1
    echo "INFO:   configure complete" | tee -a $LOGFILE
    bkgBuild "protobuf" "" "install" "$TOOLSDIR/protobuf-2.5.0/bin"
  fi
  echo " *********************************************************** " | tee -a $LOGFILE
else
  echo "INFO:  Protobuf is already installed, skipping to next tool" | tee -a $LOGFILE
fi

# -----------------------------------------------------------------------------
# install log4cxx, if not installed
if [[ !  -e /usr/lib64/liblog4cxx.so ]]; then
  cd $BASEDIR
  echo
  echo "INFO: Installing log4cxx on $(date)" | tee -a $LOGFILE
  if [ -d $TOOLSDIR/apache-log4cxx-0.10.0/lib ]; then
    echo "INFO: log4cxx is already installed, skipping to next tool" | tee -a $LOGFILE
  else
    downloadSource https://dist.apache.org/repos/dist/release/logging/log4cxx/0.10.0/apache-log4cxx-0.10.0.tar.gz apache-log4cxx-0.10.0
    cd apache-log4cxx-0.10.0
    echo "INFO:   headerfile patch - per LOG4CXX-360" | tee -a $LOGFILE
    sed -i '1 i#include <string.h>' src/main/cpp/inputstreamreader.cpp
    sed -i '1 i#include <string.h>' src/main/cpp/socketoutputstream.cpp
    sed -i '1 i#include <string.h>' src/examples/cpp/console.cpp
    sed -i '2 i#include <stdio.h>' src/examples/cpp/console.cpp
    ./configure --prefix=$TOOLSDIR/apache-log4cxx-0.10.0 >>$LOGFILE 2>&1
    echo "INFO:   configure complete" | tee -a $LOGFILE
    echo "INFO:   Be sure dependencies apr-devel and apr-util-devel are installed" | tee -a $LOGFILE
    bkgBuild "log4cxx" "" "install" "$TOOLSDIR/apache-log4cxx-0.10.0/lib"
  fi
  echo " *********************************************************** " | tee -a $LOGFILE
else
  echo "INFO:  log4cxx is already installed, skipping to next tool" | tee -a $LOGFILE
fi

# -----------------------------------------------------------------------------
# download hadoop/hdfs libs
cd $BASEDIR
echo
echo "INFO: Hadoop/HDFS libs on $(date)" | tee -a $LOGFILE
HVER="2.6.0"
if [ -d $TOOLSDIR/hadoop-${HVER} ]; then
  echo "INFO: Hadoop/HDFS is already installed, skipping to next tool" | tee -a $LOGFILE
else
  downloadSource http://archive.apache.org/dist/hadoop/common/hadoop-${HVER}/hadoop-${HVER}.tar.gz # no un-tar
  cd $TOOLSDIR
  tar -xzf $BASEDIR/hadoop-${HVER}.tar.gz \
    hadoop-${HVER}/lib/native/libhadoop\*so\* \
    hadoop-${HVER}/lib/native/libhdfs\*so\* \
    hadoop-${HVER}/include/hdfs.h
  echo "INFO:   extraction complete" | tee -a $LOGFILE
fi
echo " *********************************************************** " | tee -a $LOGFILE

# -----------------------------------------------------------------------------

echo
echo "INFO: Waiting for all background builds. This might take a while." | tee -a $LOGFILE
wait
echo "INFO: Completed tools builds on $(date)" | tee -a $LOGFILE
echo "INFO: List of tools directory: " | tee -a $LOGFILE
ls $TOOLSDIR | echo | tee -a $LOGFILE
echo "`ls $TOOLSDIR`" | tee -a $LOGFILE
echo