helix-core/src/main/scripts/integration-test/testcases/helix_random_kill.test

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# test that we can insert and retrieve a simple record

# start zookeeper on localhost:2183
export TEST_NAME=helix_random_kill
source setup_env.inc

#colorful echo
red='\e[00;31m'
green='\e[00;32m'
function cecho
{
  message="$1"
  if [ -n "$message" ]; then
    color="$2"
    if [ -z "$color" ]; then
      echo "$message"
    else
      echo -e "$color$message\e[00m"
    fi
  fi
}

zookeeper_server_ports="localhost:2188"
# default datadir integration_test/var/work/zookeeper/data/1
# start the zookeeper cluster
$SCRIPT_DIR/cm_driver.py -c zookeeper -o start --zookeeper_reset --zookeeper_server_ports=${zookeeper_server_ports}  --cmdline_props="tickTime=2000;initLimit=5;syncLimit=2" 

# create storage cluster
$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addCluster storage-cluster-12345"

# $SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addCluster relay-cluster-12345"

$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addResource storage-cluster-12345 db-12345 120 MasterSlave"

$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addNode storage-cluster-12345 localhost:8901"

$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addNode storage-cluster-12345 localhost:8902"

$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addNode storage-cluster-12345 localhost:8903"

$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addNode storage-cluster-12345 localhost:8904"

$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -addNode storage-cluster-12345 localhost:8905"

$SCRIPT_DIR/cm_driver.py -c clm_console --cmdline_args="-zkSvr localhost:2188 -rebalance storage-cluster-12345 db-12345 3"

#Launch dummy process nodes
start_port=8900
for i in `seq 1 5`; do
 port=$(($start_port + $i))
# $SCRIPT_DIR/cm_driver.py -c dummy-process -o start -l "integration-test/config/log4j-info.properties" --save_process_id --component_id=$i --cmdline_args="-helixManagerType zk -zkSvr localhost:2188 -cluster storage-cluster-12345 -host localhost -port ${port}"
  $SCRIPT_DIR/cm_driver.py -c mock-health-report-process -o start -l "integration-test/config/log4j-info.properties" --save_process_id --component_id=$i --cmdline_args="-zkSvr localhost:2188 -cluster storage-cluster-12345 -host localhost -port ${port}"
done

#Launch the cluster manager
$SCRIPT_DIR/cm_driver.py -c cluster-manager -o start --jvm_args="-javaagent:integration-test/testcases/shiftone-jrat.jar -Djrat.factory=org.shiftone.jrat.provider.tree.TreeMethodHandlerFactory" -l "integration-test/config/log4j-info.properties" --cmdline_args="-zkSvr localhost:2188 -cluster storage-cluster-12345"

#sleep 6

#$SCRIPT_DIR/cm_driver.py -c cluster-manager -o stop 

echo "wait for user input to continue"
read

#verify cluster state
verifier_1_output=$VIEW_ROOT/$LOG_DIR_FROM_ROOT/verifier_1.log
$SCRIPT_DIR/cm_driver.py -c cluster-state-verifier -o start --logfile=$verifier_1_output -l "integration-test/config/log4j-info.properties" --cmdline_args="-zkSvr localhost:2188 -cluster storage-cluster-12345 -timeout 60000"

#random kill m out of n and restart
n=5
m=1
to_kill=`shuf --input-range=1-$n | head -${m}`

for i in $to_kill; do
  cecho "killing participant $i ..." $red
  $SCRIPT_DIR/cm_driver.py -c mock-health-report-process -o stop --component_id=$i
#  $SCRIPT_DIR/cm_driver.py -c dummy-process -o stop --component_id=$i
done

sleep 6
for i in $to_kill; do
  cecho "restarting participant $i ..." $green
  port=$(($start_port + $i))
  $SCRIPT_DIR/cm_driver.py -c mock-health-report-process -o start -l "integration-test/config/log4j-info.properties" --save_process_id --component_id=$i --cmdline_args="-zkSvr localhost:2188 -cluster storage-cluster-12345 -host localhost -port ${port}"
#  $SCRIPT_DIR/cm_driver.py -c dummy-process -o start -l "integration-test/config/log4j-info.properties" --save_process_id --component_id=$i --cmdline_args="-helixManagerType zk -zkSvr localhost:2188 -cluster storage-cluster-12345 -host localhost -port ${port}"
done

#verify cluster state
verifier_1_output=$VIEW_ROOT/$LOG_DIR_FROM_ROOT/verifier_1.log
$SCRIPT_DIR/cm_driver.py -c cluster-state-verifier -o start --logfile=$verifier_1_output -l "integration-test/config/log4j-info.properties" --cmdline_args="-zkSvr localhost:2188 -cluster storage-cluster-12345 -timeout 60000"

#cleanup
for i in `seq 1 5`; do
  $SCRIPT_DIR/cm_driver.py -c mock-health-report-process -o stop --component_id=$i
#  $SCRIPT_DIR/cm_driver.py -c dummy-process -o stop --component_id=$i
done

$SCRIPT_DIR/cm_driver.py -c cluster-manager -o stop
$SCRIPT_DIR/cm_driver.py -c zookeeper -o stop

echo == GREP SUCCEED ==
#find ~/EspressoLogs/ -mmin -10 -print | xargs grep succeed
grep Successful $verifier_1_output

source report_pass_fail.inc

exit $all_stat

# add 2 more nodes
#$SCRIPT_DIR/cm_driver.py -zkSvr localhost:2188 -addNode storage-cluster-12345 localhost:8905

#$SCRIPT_DIR/cm_driver.py -zkSvr localhost:2188 -addNode storage-cluster-12345 localhost:8906

# rebalance
#$SCRIPT_DIR/cm_driver.py -zkSvr localhost:2188 -rebalance storage-cluster-12345 db-12345 3

#Launch the cluster manager
#$SCRIPT_DIR/cm_driver.py -c cluster-manager -o start --cmdline_props="-zkSvr localhost:2188 -cluster storage-cluster-12345"

#sleep 5

#$SCRIPT_DIR/cm_driver.py -c cluster-manager -o stop

#verify cluster state
#$SCRIPT_DIR/cm_driver.py -c cluster-state-verifier -o start --cmdline_props="-zkSvr localhost:2188 -cluster storage-cluster-12345"

#$SCRIPT_DIR/cm_driver.py -c dummy-process -o stop

#$SCRIPT_DIR/cm_driver.py -c cluster-manager -o stop

# do not stop for now so that people can view the state
#$SCRIPT_DIR/cm_driver.py -c zookeeper -o start
#