tools/travis_watchdog.sh

#!/usr/bin/env bash
################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

HERE="`dirname \"$0\"`"				# relative
HERE="`( cd \"$HERE\" && pwd )`" 	# absolutized and normalized
if [ -z "$HERE" ] ; then
	# error; for some reason, the path is not accessible
	# to the script (e.g. permissions re-evaled after suid)
	exit 1  # fail
fi

source "${HERE}/ci/stage.sh"
source "${HERE}/ci/maven-utils.sh"

ARTIFACTS_DIR="${HERE}/artifacts"

mkdir -p $ARTIFACTS_DIR || { echo "FAILURE: cannot create log directory '${ARTIFACTS_DIR}'." ; exit 1; }

echo "Build for commit ${TRAVIS_COMMIT} of ${TRAVIS_REPO_SLUG} [build ID: ${TRAVIS_BUILD_ID}, job number: $TRAVIS_JOB_NUMBER]." | tee "${ARTIFACTS_DIR}/build_info"

# =============================================================================
# CONFIG
# =============================================================================

# Number of seconds w/o output before printing a stack trace and killing $MVN
MAX_NO_OUTPUT=${1:-900}

# Number of seconds to sleep before checking the output again
SLEEP_TIME=20

# Maximum times to retry uploading artifacts file to transfer.sh
TRANSFER_UPLOAD_MAX_RETRIES=2

# The delay between two retries to upload artifacts file to transfer.sh. The default exponential
# backoff algorithm should be too long for the last several retries.
TRANSFER_UPLOAD_RETRY_DELAY=5

WATCHDOG_ADDITIONAL_MONITORING_FILES="$ARTIFACTS_DIR/mvn-*.log"

LOG4J_PROPERTIES=${HERE}/log4j-travis.properties

PYTHON_TEST="./flink-python/dev/lint-python.sh"
PYTHON_PID="${ARTIFACTS_DIR}/watchdog.python.pid"
PYTHON_EXIT="${ARTIFACTS_DIR}/watchdog.python.exit"
PYTHON_OUT="${ARTIFACTS_DIR}/python.out"

MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${TEST})
MVN_TEST_MODULES=$(get_test_modules_for_stage ${TEST})

# Maven command to run. We set the forkCount manually, because otherwise Maven sees too many cores
# on the Travis VMs. Set forkCountTestPackage to 1 for container-based environment (4 GiB memory)
# and 2 for sudo-enabled environment (7.5 GiB memory).
MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES"
MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Pskip-webui-build $MVN_LOGGING_OPTIONS"
MVN_COMPILE_OPTIONS="-DskipTests"
MVN_TEST_OPTIONS="-Dflink.tests.with-openssl"

e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')

MVN_COMPILE="run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install"
MVN_TEST="run_mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS $PROFILE $MVN_TEST_MODULES verify"
# don't move the e2e-pre-commit profile activation into the misc entry in .travis.yml, since it breaks caching
MVN_E2E="run_mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS -Pe2e-pre-commit -pl ${e2e_modules},flink-dist verify"

MVN_PID="${ARTIFACTS_DIR}/watchdog.mvn.pid"
MVN_EXIT="${ARTIFACTS_DIR}/watchdog.mvn.exit"
MVN_OUT="${ARTIFACTS_DIR}/mvn.out"

TRACE_OUT="${ARTIFACTS_DIR}/jps-traces.out"

# E.g. travis-artifacts/apache/flink/1595/1595.1
UPLOAD_TARGET_PATH="travis-artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BUILD_NUMBER}/"
# These variables are stored as secure variables in '.travis.yml', which are generated per repo via
# the travis command line tool.
UPLOAD_BUCKET=$ARTIFACTS_AWS_BUCKET
UPLOAD_ACCESS_KEY=$ARTIFACTS_AWS_ACCESS_KEY
UPLOAD_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY

ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz

if [ ! -z "$TF_BUILD" ] ; then
	# set proper artifacts file name on Azure Pipelines
	ARTIFACTS_FILE=${BUILD_BUILDNUMBER}.tar.gz
fi

# enable coredumps
ulimit -c unlimited
export JAVA_TOOL_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"

if [ $TEST == $STAGE_PYTHON ]; then
	CMD=$PYTHON_TEST
	CMD_PID=$PYTHON_PID
	CMD_OUT=$PYTHON_OUT
	CMD_EXIT=$PYTHON_EXIT
	CMD_TYPE="PYTHON"
else
	CMD=$MVN_COMPILE
	CMD_PID=$MVN_PID
	CMD_OUT=$MVN_OUT
	CMD_EXIT=$MVN_EXIT
	CMD_TYPE="MVN"
fi

# =============================================================================
# FUNCTIONS
# =============================================================================

upload_artifacts_s3() {
	echo "PRODUCED build artifacts."

	ls $ARTIFACTS_DIR

	echo "COMPRESSING build artifacts."

	cd $ARTIFACTS_DIR
	dmesg > container.log
	tar -zcvf $ARTIFACTS_FILE *

	# Upload to secured S3
	if [ -n "$UPLOAD_BUCKET" ] && [ -n "$UPLOAD_ACCESS_KEY" ] && [ -n "$UPLOAD_SECRET_KEY" ]; then

		# Install artifacts tool
		curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash

		PATH=$HOME/bin/artifacts:$HOME/bin:$PATH

		echo "UPLOADING build artifacts."

		# Upload everything in $ARTIFACTS_DIR. Use relative path, otherwise the upload tool
		# re-creates the whole directory structure from root.
		artifacts upload --bucket $UPLOAD_BUCKET --key $UPLOAD_ACCESS_KEY --secret $UPLOAD_SECRET_KEY --target-paths $UPLOAD_TARGET_PATH $ARTIFACTS_FILE
	fi

	# On Azure, publish ARTIFACTS_FILE as a build artifact
	if [ ! -z "$TF_BUILD" ] ; then
		TIMESTAMP=`date +%s` # append timestamp to name to allow multiple uploads for the same module
		ARTIFACT_DIR="$(pwd)/artifact-dir"
		mkdir $ARTIFACT_DIR
		cp $ARTIFACTS_FILE $ARTIFACT_DIR/
		
		echo "##vso[task.setvariable variable=ARTIFACT_DIR]$ARTIFACT_DIR"
		echo "##vso[task.setvariable variable=ARTIFACT_NAME]$(echo $MODULE | tr -dc '[:alnum:]\n\r')-$TIMESTAMP"
	fi

	# upload to https://transfer.sh
	echo "Uploading to transfer.sh"
	curl --retry ${TRANSFER_UPLOAD_MAX_RETRIES} --retry-delay ${TRANSFER_UPLOAD_RETRY_DELAY} --upload-file $ARTIFACTS_FILE --max-time 60 https://transfer.sh
}

print_stacktraces () {
	echo "=============================================================================="
	echo "The following Java processes are running (JPS)"
	echo "=============================================================================="

	jps

	local pids=( $(jps | awk '{print $1}') )

	for pid in "${pids[@]}"; do
		echo "=============================================================================="
		echo "Printing stack trace of Java process ${pid}"
		echo "=============================================================================="

		jstack $pid
	done
}

# locate YARN logs and put them into artifacts directory
put_yarn_logs_to_artifacts() {
	# Make sure to be in project root
	cd $HERE/../
	for file in `find ./flink-yarn-tests/target -type f -name '*.log' -or -name '*.out'`; do
		TARGET_FILE=`echo "$file" | grep -Eo "container_[0-9_]+/(.*).[a-z]{3}"`
		TARGET_DIR=`dirname	 "$TARGET_FILE"`
		mkdir -p "$ARTIFACTS_DIR/yarn-tests/$TARGET_DIR"
		cp $file "$ARTIFACTS_DIR/yarn-tests/$TARGET_FILE"
	done
}

max_of() {
  local max number

  max="$1"

  for number in "${@:2}"; do
    if ((number > max)); then
      max="$number"
    fi
  done

  printf '%d\n' "$max"
}

# Returns the highest modification time out of $CMD_OUT (which is the command output file)
# and any file(s) named "mvn-*.log" (which are logging files created by Flink's tests)
mod_time () {
	CMD_OUT_MOD_TIME=`stat -c "%Y" $CMD_OUT`
	ADDITIONAL_FILES_MOD_TIMES=`stat -c "%Y" $WATCHDOG_ADDITIONAL_MONITORING_FILES 2> /dev/null`
	echo `max_of $CMD_OUT_MOD_TIME $ADDITIONAL_FILES_MOD_TIMES`
}

the_time() {
	echo `date +%s`
}

# =============================================================================
# WATCHDOG
# =============================================================================

watchdog () {
	touch $CMD_OUT

	while true; do
		sleep $SLEEP_TIME

		time_diff=$((`the_time` - `mod_time`))

		if [ $time_diff -ge $MAX_NO_OUTPUT ]; then
			echo "=============================================================================="
			echo "Maven produced no output for ${MAX_NO_OUTPUT} seconds."
			echo "=============================================================================="

			print_stacktraces | tee $TRACE_OUT

			# Kill $CMD and all descendants
			pkill -P $(<$CMD_PID)

			exit 1
		fi
	done
}

run_with_watchdog() {
	local cmd="$1"

	watchdog &
	WD_PID=$!
	echo "STARTED watchdog (${WD_PID})."

	# Make sure to be in project root
	cd "$HERE/../"

	echo "RUNNING '${cmd}'."

	# Run $CMD and pipe output to $CMD_OUT for the watchdog. The PID is written to $CMD_PID to
	# allow the watchdog to kill $CMD if it is not producing any output anymore. $CMD_EXIT contains
	# the exit code. This is important for Travis' build life-cycle (success/failure).
	( $cmd & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$CMD_PID 4>$CMD_EXIT | tee $CMD_OUT

	EXIT_CODE=$(<$CMD_EXIT)

	echo "${CMD_TYPE} exited with EXIT CODE: ${EXIT_CODE}."

	# Make sure to kill the watchdog in any case after $CMD has completed
	echo "Trying to KILL watchdog (${WD_PID})."
	( kill $WD_PID 2>&1 ) > /dev/null

	rm $CMD_PID
	rm $CMD_EXIT
}

run_with_watchdog "$CMD"

# Run tests if compilation was successful
if [ $CMD_TYPE == "MVN" ]; then
	if [ $EXIT_CODE == 0 ]; then
		run_with_watchdog "$MVN_TEST"
	else
		echo "=============================================================================="
		echo "Compilation failure detected, skipping test execution."
		echo "=============================================================================="
	fi
fi

# Post

# only misc builds flink-dist and flink-yarn-tests
case $TEST in
	(misc)
		put_yarn_logs_to_artifacts
	;;
esac

collect_coredumps `pwd` $ARTIFACTS_DIR

upload_artifacts_s3

# since we are in flink/tools/artifacts
# we are going back to
cd ../../

# only run end-to-end tests in misc because we only have flink-dist here
case $TEST in
    (misc)
        # If we are not on Azure (we are on Travis) run precommit tests in misc stage.
        # On Azure, we run them in a separate job
        if [ -z "$TF_BUILD" ] ; then
            if [ $EXIT_CODE == 0 ]; then
                echo "\n\n==============================================================================\n"
                echo "Running bash end-to-end tests\n"
                echo "==============================================================================\n"

                FLINK_DIR=build-target flink-end-to-end-tests/run-pre-commit-tests.sh

                EXIT_CODE=$?
            else
                echo "\n==============================================================================\n"
                echo "Previous build failure detected, skipping bash end-to-end tests.\n"
                echo "==============================================================================\n"
            fi
	        if [ $EXIT_CODE == 0 ]; then
	            echo "\n\n==============================================================================\n"
	            echo "Running java end-to-end tests\n"
	            echo "==============================================================================\n"

	            run_with_watchdog "$MVN_E2E -DdistDir=$(readlink -e build-target)"
	        else
	            echo "\n==============================================================================\n"
	            echo "Previous build failure detected, skipping java end-to-end tests.\n"
	        fi
	    fi
    ;;
esac

# Exit code for Travis build success/failure
exit $EXIT_CODE