bin/run-all-tests-timeout-check.sh

#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script run by run-all-tests.sh that checks every 60 sec if the timeout expired and on
# timeout prints the stacktraces of all impalads and then finally kills running tests.
# Takes the timeout in minutes as an argument.

SLEEP_TIMEOUT_S=0
if [ -z "$1" ]; then
  echo "Expected timeout value as an argument"
  exit 1
else
  SLEEP_TIMEOUT_S=$(($1 * 60))
fi

[[ $SLEEP_TIMEOUT_S < 1 ]] && exit

echo
echo
echo "**** Timout Timer Started (pid $$, ppid $PPID) for $SLEEP_TIMEOUT_S s! ****"
echo
echo

# Check timer every 60 seconds and only proceed if the parent process is still alive.
# Note: $SECONDS is a bash built-in that counts seconds since bash started.
while ((SLEEP_TIMEOUT_S - SECONDS > 0)); do
  sleep 1
  if ! ps $PPID &> /dev/null; then
    echo "Timeout Timer Exited because $PPID is gone."
    exit
  fi
done

echo
echo
echo '**** Tests TIMED OUT! ****'
echo
echo

# Impala probably has a thread stuck. Print the stacktrace to the console output.
mkdir -p "$IMPALA_TIMEOUT_LOGS_DIR"
for pid in $(pgrep impalad); do
  echo "**** Generating stacktrace of impalad with process id: $pid ****"
  gdb -ex "thread apply all bt"  --batch -p $pid > "${IMPALA_TIMEOUT_LOGS_DIR}/${pid}.txt"
done

# Now kill any running tests.
kill $PPID

"${IMPALA_HOME}"/bin/generate_junitxml.py --step "test_run" --error "Test run timed out.
This probably happened due to a hung thread which can be confirmed by looking at the
stacktrace of running impalad processes at ${IMPALA_TIMEOUT_LOGS_DIR}"