core/sqf/monitor/test/runtest

#!/bin/sh
#
# @@@ START COPYRIGHT @@@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# @@@ END COPYRIGHT @@@
#
# Execute this test script from $TRAF_HOME/monitor/test.
#
# Cluster environment setup:
#    - create directories
#      cd $TRAF_HOME
#      pdsh $MY_NODES mkdir $PWD/monitor
#      pdsh $MY_NODES mkdir $PWD/monitor/test
#    - copy test files to $TRAF_HOME/monitor/test in all nodes
#      cd $TRAF_HOME/monitor/test
#      pdcp $MY_NODES -p -r /home/sqft2/monitor/test/* $PWD/
#    - execute runtest -cluster
#      cd $TRAF_HOME/monitor/test
#      runtest -cluster
#
# Virtual environment setup:
#    - execute runtest -virtual
#      cd $TRAF_HOME/monitor/test
#      runtest -virtual
#

function Usage
{
    echo
    echo "runtest { -cluster | -virtual } [ -nogen | -trace | -test <num> ]"
    echo
    echo "Where: <num> is one of the following tests:"
    echo "         1     - Child Exit"
    echo "         2     - Multi-Node"
    echo "         3     - Registry"
    echo "         4     - Death Notice"
    echo "         5     - Persistent Process"
    echo "         6     - DTM Process"
    echo "         7     - SPX Process"
    echo "         8     - Process Create"
    echo "         9     - Node down before startup"
    echo "        10     - TmSync"
    echo
}

if [ $# -lt 1 ]; then
    Usage
    exit 1
fi

cluster=
conf=
virtual=
trace=
nogen=

declare -i test

let test=-1

if [ "$1" = "-cluster" ]; then
    cluster=-cluster
    conf="sqconfig.monitor.cluster"
    shift
fi
if [ "$1" = "-virtual" ]; then
    virtual=-virtual
    conf="sqconfig.monitor.virtual"
    shift
fi
if [ "$1" = "-nogen" ]; then
    nogen=${1}
    shift
fi
if [ "$1" = "-trace" ]; then
    trace=-t
    shift
fi
if [ "$1" = "-test" ]; then
    shift
    test=${1}
    shift
fi

#
# Setup test execution
#
export PATH=$PATH:$PWD/Linux-x86_64/64/dbg
cd $TRAF_HOME/monitor/test
echo $PWD

if [ "$cluster" = "-cluster" ]; then
    ckillall
else if [ "$virtual" = "-virtual" ]; then
    ckillall
else
    Usage
    exit 1
fi
fi

if [ ! -z "$SQ_DTM_PERSISTENT_PROCESS" ]; then
    let dtm_persistent_process=`echo $SQ_DTM_PERSISTENT_PROCESS`
else
    let dtm_persistent_process="0"
fi

echo ************ Executing test $test ************

export SQ_SHELL_NOCWD=1
# Uncomment to enable LIO tracing in test programs
#export SQ_LOCAL_IO_SHELL_TRACE=1

#
# Cleanup
#
sqipcrm
if [ "$cluster" = "-cluster" ]; then
    pdsh $MY_NODES "rm -f $MPI_TMPDIR/monitor.port.*"
    pdsh $MY_NODES "rm $PWD/core* $PWD/*.lst $PWD/stdout_*"
else
    rm -f $MPI_TMPDIR/monitor.port.*
    rm core* *.lst stdout_*
    # Establish SQ virtual cluster parameters
    export SQ_VIRTUAL_NODES=6
    export SQ_VIRTUAL_NID=0
fi

#
# Setup up monitor configuration
#
if [ "$nogen" = "-nogen" ]; then
    echo "sqgen not executed!"
else
    echo sqgen $TRAF_HOME/monitor/test/$conf
    sqgen $TRAF_HOME/monitor/test/$conf
fi

#
# Execute tests
#
if ( [ $test '==' -1 ] || [ $test '==' 1 ] ); then
 echo "***"
 echo "*** Executing Child Exit test"
 echo "***"
shell <<eof
 startup
 exec {name \$CTRLR, nid 0, out $TRAF_HOME/monitor/test/childExit.lst} childExitCtrl $trace
 shutdown
 exit
eof
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/childExit.lst
     pdsh $MY_NODES grep FAILED $PWD/childExit.lst
 else
     grep PASSED $PWD/childExit.lst
     grep FAILED $PWD/childExit.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 2 ] ); then
 echo "***"
 echo "*** Executing Multi-Node test"
 echo "***"
shell <<eof
 startup
 ! General tests for monitor functionality. This includes
 !  Open, Close, NewProcess, ProcessInfo and sync node state
 !  between nodes.
 node
 ! To enable tracing add "-t" argument to the end of the "exec" line
 exec {nowait,name \$SERV0,nid 0} server $trace
 exec {nowait,pri 5,name \$SERV1} server $trace
 ps
 exec {pri 10,name \$CLIENT,nid 0, out $TRAF_HOME/monitor/test/multiNode.lst} client $trace
 shutdown
 exit
eof
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/multiNode.lst
     pdsh $MY_NODES grep FAILED $PWD/multiNode.lst
 else
     grep PASSED $PWD/multiNode.lst
     grep FAILED $PWD/multiNode.lst
 fi
fi

 if ( [ $test '==' -1 ] || [ $test '==' 3 ] ); then
 echo "***"
 echo "*** Executing Registry test"
 echo "***"
shell <<eof
 startup
 exec {name \$CTRLR, nid 0, out $TRAF_HOME/monitor/test/regTest.lst} regTestCtrl $trace
 shutdown
 exit
eof
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/regTest.lst
     pdsh $MY_NODES grep FAILED $PWD/regTest.lst
 else
     grep PASSED $PWD/regTest.lst
     grep FAILED $PWD/regTest.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 4 ] ); then
 echo "***"
 echo "*** Executing Death Notice test"
 echo "***"
 shell <<eof
 startup
 ! Test to register and cancel process death
 ! notices from non-parent process.
 !
 exec {nowait,name \$serv0, out serv0.lst} server waitforclose $trace
 exec {nowait,name \$serv1, out serv1.lst} server waitforclose $trace
 exec {nowait,nid 0,name \$TM00,out tm00.lst} server waitforclose $trace
 exec {nowait,nid 1,name \$TM01,out tm01.lst} server waitforclose $trace
 exec {nowait,nid 2,name \$TM02,out tm02.lst} server waitforclose $trace
 ps
 ! To enable tracing add "-t" argument to the end of the "exec" line
 !
 exec {name \$DEATH, nid 0, out $TRAF_HOME/monitor/test/deathNotice.lst} deathNotice $trace
 shutdown
 exit
eof
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/deathNotice.lst
     pdsh $MY_NODES grep FAILED $PWD/deathNotice.lst
 else
     grep PASSED $PWD/deathNotice.lst
     grep FAILED $PWD/deathNotice.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 5 ] ); then
 if [ "$cluster" = "-cluster" ]; then
  echo "***"
  echo "*** Executing Persistent Process cluster test"
  echo "***"
shell <<eof
 startup
 down 2 !
 delay 15
 exec {name \$PPROC, nid 0, out $TRAF_HOME/monitor/test/persistentProc.lst} persistentProc $trace
 shutdown
 exit
eof
  sleep 1
  cstat -h
  pdsh $MY_NODES grep PASSED $PWD/persistentProc.lst
  pdsh $MY_NODES grep FAILED $PWD/persistentProc.lst
 else
  echo "***"
  echo "*** Executing Persistent Process test"
  echo "***"
shell <<eof
 startup
 exec {name \$PPROC, nid 0, out $TRAF_HOME/monitor/test/persistentProc.lst} persistentProc $trace
 shutdown
 exit
eof
  sleep 1
  cstat -h
  grep PASSED $PWD/persistentProc.lst
  grep FAILED $PWD/persistentProc.lst
 fi
fi

if ( [[ $dtm_persistent_process == "1" ]] &&
      ([ $test '==' -1 ] || [ $test '==' 6 ]) ); then
 echo "***"
 echo "*** Executing DTM Process test"
 echo "***"
shell <<eof
 startup
 exec {name \$DTMCTRL, nid 0, out $TRAF_HOME/monitor/test/dtmTest.lst} dtmCtrl $trace
 shutdown
 exit
eof
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/dtmTest.lst
     pdsh $MY_NODES grep FAILED $PWD/dtmTest.lst
 else
     grep PASSED $PWD/dtmTest.lst
     grep FAILED $PWD/dtmTest.lst
 fi
else
 echo "DTM is not configured as a persistent process. Skipping test 6!"
fi

if ( [ $test '==' -1 ] || [ $test '==' 7 ] ); then
 echo "***"
 echo "*** Executing SPX Process test"
 echo "***"
shell <<eof
 startup
 exec {name \$SPXCTRL, nid 0, out $TRAF_HOME/monitor/test/spxTest.lst} spxCtrl $trace
 shutdown
 exit
eof
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/spxTest.lst
     pdsh $MY_NODES grep FAILED $PWD/spxTest.lst
 else
     grep PASSED $PWD/spxTest.lst
     grep FAILED $PWD/spxTest.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 8 ] ); then
 echo "***"
 echo "*** Executing Process Create test"
 echo "***"
shell <<eof
 startup
 exec {name \$PCRE8, nid 0, out $TRAF_HOME/monitor/test/procCreate.lst} procCreate $trace -x
 exit
eof
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/procCreate.lst
     pdsh $MY_NODES grep FAILED $PWD/procCreate.lst
 else
     grep PASSED $PWD/procCreate.lst
     grep FAILED $PWD/procCreate.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 9 ] ); then
 echo "***"
 echo "*** Executing Process Creates Node down before startup test"
 echo "***"
shell <<eof 
 startup
 exec {nowait, name \$PCRE8, nid 0, out $TRAF_HOME/monitor/test/nodeDown.lst} procCreate $trace -y
 delay 3
 down 1 !
 delay 10
 shutdown
 exit
eof
 sleep 10
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/nodeDown.lst
     pdsh $MY_NODES grep FAILED $PWD/nodeDown.lst
 else
     grep PASSED $PWD/nodeDown.lst
     grep FAILED $PWD/nodeDown.lst
 fi
fi


#if ( [ $test '==' -1 ] || [ $test '==' 9 ] ); then
if ( [ $test '==' 9 ] ); then
 if [ "$cluster" = "-cluster" ]; then
  echo "***"
  echo "*** Executing TmSync cluster test (subtest 7 disabled until it's fixed)"
  echo "***"
shell <<eof
 startup
 exec {nowait, nid 0, name \$CTRLR, out $TRAF_HOME/monitor/test/tmSync.lst} tmSyncCtrl -n 1,3,4,5,6 $trace
 wait $CTRLR
 shutdown
 exit
eof
  sleep 1
  cstat -h
  pdsh $MY_NODES grep PASSED $PWD/tmSync.lst
  pdsh $MY_NODES grep FAILED $PWD/tmSync.lst
  echo "***"
  echo "*** Executing TmSync cluster test 10"
  echo "***"
shell <<eof
 startup
 exec {nowait, nid 0, name \$CTRLR, out $TRAF_HOME/monitor/test/tmSync8.lst} tmSyncCtrl -n 8 $trace
 wait $CTRLR
 shutdown
 exit
eof
  sleep 1
  cstat -h
  pdsh $MY_NODES grep PASSED $PWD/tmSync9.lst
  pdsh $MY_NODES grep FAILED $PWD/tmSync9.lst
  echo "***"
  echo "*** Executing TmSync cluster test 10"
  echo "***"
shell <<eof
 startup
 down 2 !
 delay 15
 exec {nowait, nid 0, name \$CTRLR, out $TRAF_HOME/monitor/test/tmSync10.lst} tmSyncCtrl -n 10 $trace
 wait $CTRLR
 shutdown
 exit
eof
  sleep 1
  cstat -h
  pdsh $MY_NODES grep PASSED $PWD/tmSync10.lst
  pdsh $MY_NODES grep FAILED $PWD/tmSync10.lst
 else
  echo "***"
  echo "*** Executing TmSync test"
  echo "***"
shell <<eof
 startup
 exec {nowait, nid 0, name \$CTRLR, out $TRAF_HOME/monitor/test/tmSync.lst} tmSyncCtrl -n 3,4,5,6,7 $trace
 wait $CTRLR
 shutdown
 exit
eof
  # Reset SQ virtual cluster parameters
  unset SQ_VIRTUAL_NODES
  unset SQ_VIRTUAL_NID
 fi
 
 sleep 1
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/tmSync.lst
     pdsh $MY_NODES grep FAILED $PWD/tmSync.lst
 else
     grep PASSED $PWD/tmSync.lst
     grep FAILED $PWD/tmSync.lst
 fi
fi

if ( [ $test '==' -1 ] ); then
 echo "***"
 echo "*** Monitor Test Results"
 echo "***"
 if [ "$cluster" = "-cluster" ]; then
     pdsh $MY_NODES grep PASSED $PWD/childExit.lst
     pdsh $MY_NODES grep FAILED $PWD/childExit.lst
     pdsh $MY_NODES grep PASSED $PWD/multiNode.lst
     pdsh $MY_NODES grep FAILED $PWD/multiNode.lst
     pdsh $MY_NODES grep PASSED $PWD/regTest.lst
     pdsh $MY_NODES grep FAILED $PWD/regTest.lst
     pdsh $MY_NODES grep PASSED $PWD/deathNotice.lst
     pdsh $MY_NODES grep FAILED $PWD/deathNotice.lst
     pdsh $MY_NODES grep PASSED $PWD/persistentProc.lst
     pdsh $MY_NODES grep FAILED $PWD/persistentProc.lst
     pdsh $MY_NODES grep PASSED $PWD/dtmTest.lst
     pdsh $MY_NODES grep FAILED $PWD/dtmTest.lst
     pdsh $MY_NODES grep PASSED $PWD/spxTest.lst
     pdsh $MY_NODES grep FAILED $PWD/spxTest.lst
     pdsh $MY_NODES grep PASSED $PWD/procCreate.lst
     pdsh $MY_NODES grep FAILED $PWD/procCreate.lst
     pdsh $MY_NODES grep PASSED $PWD/nodeDown.lst
     pdsh $MY_NODES grep FAILED $PWD/nodeDown.lst
#     pdsh $MY_NODES grep PASSED $PWD/tmSync.lst
#     pdsh $MY_NODES grep FAILED $PWD/tmSync.lst
#     pdsh $MY_NODES grep PASSED $PWD/tmSync9.lst
#     pdsh $MY_NODES grep FAILED $PWD/tmSync9.lst
#     pdsh $MY_NODES grep PASSED $PWD/tmSync10.lst
#     pdsh $MY_NODES grep FAILED $PWD/tmSync10.lst
 else
     grep PASSED $PWD/childExit.lst
     grep FAILED $PWD/childExit.lst
     grep PASSED $PWD/multiNode.lst
     grep FAILED $PWD/multiNode.lst
     grep PASSED $PWD/regTest.lst
     grep FAILED $PWD/regTest.lst
     grep PASSED $PWD/deathNotice.lst
     grep FAILED $PWD/deathNotice.lst
     grep PASSED $PWD/persistentProc.lst
     grep FAILED $PWD/persistentProc.lst
     grep PASSED $PWD/dtmTest.lst
     grep FAILED $PWD/dtmTest.lst
     grep PASSED $PWD/spxTest.lst
     grep FAILED $PWD/spxTest.lst
     grep PASSED $PWD/procCreate.lst
     grep FAILED $PWD/procCreate.lst
     grep PASSED $PWD/nodeDown.lst
     grep FAILED $PWD/nodeDown.lst
#     grep PASSED $PWD/tmSync.lst
#     grep FAILED $PWD/tmSync.lst
 fi
fi