core/sqf/sql/scripts/sqstop

#!/bin/bash
#
# @@@ START COPYRIGHT @@@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# @@@ END COPYRIGHT @@@

# Perform an SQ shutdown 

function Usage {

    echo 
    echo "Usage: $0 [ abrupt | immediate ]"
    echo 
    echo "This command is used to perform a shutdown of the SQ environment."
    echo "If a parameter is not specified, then a normal shutdown is performed."
    echo

}

function LogIt {
    echo  "`date`: $1" >> $TRAF_HOME/logs/startup.log
}

function echoLog {

    echo $1 | tee -a $SQMON_LOG
}

script_name=`/bin/basename $0`
SQLOG_DIR=$TRAF_HOME/logs
SQMON_LOG=$SQLOG_DIR/sqmon.log
SQGOMON_FILE=$TRAF_HOME/sql/scripts/gomon.cold
SQSTOP_EXIT_STATUS=$TRAF_HOME/sql/scripts/sqstop.exit_status

if [ $# -eq 0 ]; then
    shutdowntype=normal
    sparam=
else
    if ( 
	    [ $1 != "abrupt" ] &&
	    [ $1 != "immediate" ]
	); then
	echo "Invalid parameter: $1";
	Usage;
	exit 1;
    else
	shutdowntype=$1;
	sparam=$1;
    fi
fi

LogIt "Begin $script_name $shutdowntype"

if [[ ! -e $SQGOMON_FILE ]]; then 
    echo "Unable to find file: gomon.cold.  An sqgen may need to be run"
    exit 0;
fi

# Check if the SQ environment is up or not
sqcheck -i 1 -d 1 > /dev/null 2>&1
sq_stat=$?
if [[ $sq_stat -eq 255 ]]; then
    lv_msg="SQ environment is not up."
    echo "$lv_msg"
    LogIt "End $script_name $shutdowntype, exit code: 0, $lv_msg"
    exit 0;
fi

rm -f $SQSTOP_EXIT_STATUS

# Stop the REST Service
reststop

# Stop the DCS Service
dcsstop

# Stop the lob Service
#lobstop

# Issue the shutdown request
echoLog "Shutting down ($shutdowntype) the SQ environment!"
echoLog "`date`"
sqshell -a <<eof
ps
shutdown $sparam
eof

echoLog "Issued a 'shutdown $shutdowntype' request"


sleep 2

# Now check whether the SQ environment is still up or not
# try to attach the SQ shell, if it fails, then it means 
# that the environment is down.

declare -i lv_num_checks
declare -i lv_max_checks
declare -i lv_done
declare -i lv_process_count_curr
declare -i lv_process_count_last
declare -i lv_displayed_process_still_executing
declare -i lv_exit_status

let lv_num_checks=0
let lv_max_checks=200
let lv_done=0
let lv_process_count_curr=0
let lv_process_count_last=0
let lv_displayed_process_still_executing=0


bFirst=1
sq_stat=0
echoLog
echoLog "Shutdown in progress";

echoLog


while [[ $sq_stat == 0 ]]; do

    sleep 5

    # Count the number of processes
    lv_process_count_curr=`cstat 2>/dev/null | grep -v "\-\-\-   \-\-\-\-" | grep -v 'pid   ppid' | wc -l`
    let sq_stat=(lv_process_count_curr '==' 0)
    let lv_cnt_check=(lv_process_count_curr '<' lv_process_count_last)
    if [ $lv_cnt_check '==' 1 ]; then
	let lv_num_checks=0
    fi
    lv_process_count_last=lv_process_count_curr
    
    if [ $lv_num_checks '==' 0 ]; then
	echo -ne "\r# of SQ processes: $lv_process_count_last"
    else
	echo -n "."
    fi

    let ++lv_num_checks

    if [[ $lv_num_checks -eq 100 ]]; then 
	    date  >> $SQMON_LOG
	    cstat >> $SQMON_LOG
    fi
    let lv_done=(lv_num_checks '>' lv_max_checks)
    if [ $lv_done '==' 1 ]; then
        if [ $lv_displayed_process_still_executing '==' 0 ]; then
	    let lv_displayed_process_still_executing=1
	    echoLog 
	    echoLog "These are the processes still running. Please check if any application process (such as sqlci) is still running that might hamper shutdown"
	    cstat | tee -a $SQMON_LOG
	    let lv_num_checks=0
	else
	    lv_shutdown_status=2
	    echoLog "`date`"
	    lv_msg="SQ Shutdown is taking longer than usual and seems to be stalled at $lv_process_count_curr processes. "
	    echoLog "$lv_msg"
	    cstat >> $SQMON_LOG
	    echoLog "Exiting."
	    let lv_exit_status=1
	    LogIt "End $script_name $shutdowntype, exit code: $lv_exit_status, $lv_msg"
	    echo "$lv_exit_status" > $SQSTOP_EXIT_STATUS
	    exit $lv_exit_status
	fi
    fi

done
echoLog

shutdownmsg="SQ Shutdown ($shutdowntype) from $PWD Successful"

if [ -e $SQ_PDSH ]; then
   L_PDSH="$PDSH $MY_NODES $PDSH_SSH_CMD"
else
   L_PDSH=
fi
$L_PDSH rm -f /dev/shm/sem.rms* 2>/dev/null

# Clean up Floating IP bound
if (test -n "${CLUSTERNAME}"); then
	# Check whether floating ip is configured.
	gv_floating_ip=`grep -i "begin floating_ip" $TRAF_HOME/sql/scripts/sqconfig | grep -v '^#'`
	if [ -n "$gv_floating_ip" ]; then
		echoLog "Clean up Floating IP..."

		cmd_timeout="-u 30"

		# Get floating ip addresses.
		gv_float_external_ip=`sed -n '/begin floating_ip/,/end floating_ip/p' $TRAF_HOME/sql/scripts/sqconfig | grep -v '^#' | grep external-ip | awk 'BEGIN{FS=";"}{split($3, ary, "="); print ary[2];}'`
		gv_float_external_interface=`sed -n '/begin floating_ip/,/end floating_ip/p' $TRAF_HOME/sql/scripts/sqconfig | grep -v '^#' | grep external-ip | awk 'BEGIN{FS=";"}{split($2, ary, "="); print ary[2];}'`
		gv_float_internal_ip=`sed -n '/begin floating_ip/,/end floating_ip/p' $TRAF_HOME/sql/scripts/sqconfig | grep -v '^#' | grep internal-ip | awk 'BEGIN{FS=";"}{split($3, ary, "="); print ary[2];}'`
		gv_float_internal_interface=`sed -n '/begin floating_ip/,/end floating_ip/p' $TRAF_HOME/sql/scripts/sqconfig | grep -v '^#' | grep internal-ip | awk 'BEGIN{FS=";"}{split($2, ary, "="); print ary[2];}'`

		# Check and get all the nodes that the ext/int IP address have been set. And do the unbind.
		
		# Unbind External IP.
		for curnode in `pdsh $cmd_timeout $MY_NODES ip addr show | grep $gv_float_external_ip | awk -F' ' '/^.+:[[:space:]]+.*/ {print $1;}' | cut -d':' -f1 | sed '/^$/d'`; do
			for myinterface in `pdsh $cmd_timeout -N -w $curnode ip link show | awk -F': ' '/^[0-9]+:.*/ {print $2}'`; do
				# Check whether the ext/int ip addresses are bound to any interface already.
				# We delete the IP address anyway, no matter which interface and port number it is bound on.
				match_ip_interface=`pdsh $cmd_timeout -N -w $curnode ip addr show $myinterface | grep $gv_float_external_ip`
				if [ -n "$match_ip_interface" ]; then
					unbindip=`echo "$match_ip_interface"|awk '{print $2}'`
					unbindlb=`echo "$match_ip_interface"|awk '{print $NF}'`
					echoLog "pdsh $cmd_timeout -S -w $curnode sudo ip addr del $unbindip dev $myinterface label $unbindlb"
					pdsh $cmd_timeout -S -w $curnode sudo ip addr del $unbindip dev $myinterface label $unbindlb >> $TRAF_HOME/logs/ndcsunbind.log
				fi
	       done
	   done
		
	   # Unbind Internal IP.
		for curnode in `pdsh $cmd_timeout $MY_NODES ip addr show | grep $gv_float_internal_ip | awk -F' ' '/^.+:[[:space:]]+.*/ {print $1;}' | cut -d':' -f1 | sed '/^$/d'`; do
			for myinterface in `pdsh $cmd_timeout -N -w $curnode ip link show | awk -F': ' '/^[0-9]+:.*/ {print $2}'`; do
				# Check whether the ext/int ip addresses are bound to any interface already.
				# We delete the IP address anyway, no matter which interface and port number it is bound on.
				match_ip_interface=`pdsh $cmd_timeout -N -w $curnode ip addr show $myinterface | grep $gv_float_internal_ip`
				if [ -n "$match_ip_interface" ]; then
					unbindip=`echo "$match_ip_interface"|awk '{print $2}'`
					unbindlb=`echo "$match_ip_interface"|awk '{print $NF}'`
					echoLog "pdsh $cmd_timeout -S -w $curnode sudo ip addr del $unbindip dev $myinterface label $unbindlb"
					pdsh $cmd_timeout -S -w $curnode sudo ip addr del $unbindip dev $myinterface label $unbindlb >> $TRAF_HOME/logs/ndcsunbind.log
			   fi
		   done
	   done
   fi
fi

echoLog "$shutdownmsg"
echoLog "`date`"

let lv_exit_status=0
LogIt "End $script_name $shutdowntype, exit code: $lv_exit_status"
echo "$lv_exit_status" > $SQSTOP_EXIT_STATUS
exit $lv_exit_status