tools/azure-pipelines/e2e_uploading_watchdog.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file has the following tasks
# a) It reads the e2e timeout from the configuration file
# b) It prints a warning if the test has reached 80% of it's execution time
# c) N minutes before the end of the execution time, it will start uploading the current output as azure artifacts

COMMAND=$1

HERE="`dirname \"$0\"`"             # relative
HERE="`( cd \"$HERE\" && pwd )`"    # absolutized and normalized
if [ -z "$HERE" ] ; then
	exit 1
fi

OUTPUT_FILE=/tmp/_e2e_watchdog.output
# Start uploading 11 minutes before the timeout imposed by Azure (11 minutes because the upload happens 
# every 5 minutes, so we should ideally get 2 uploads and then the operation gets killed)
START_LOG_UPLOAD_SECONDS_FROM_END=$((11*60))

DEFINED_TIMEOUT_MINUTES=`cat $HERE/jobs-template.yml | grep "timeoutInMinutes" | tail -n 1 | cut -d ":" -f 2 | tr -d '[:space:]'`
DEFINED_TIMEOUT_SECONDS=$(($DEFINED_TIMEOUT_MINUTES*60))

echo "Running command '$COMMAND' with a timeout of $DEFINED_TIMEOUT_MINUTES minutes."

function warning_watchdog {
	SLEEP_TIME=$(echo "scale=0; $DEFINED_TIMEOUT_SECONDS*0.8/1" | bc)
	sleep $SLEEP_TIME
	echo "=========================================================================================="
	echo "=== WARNING: This E2E Run took already 80% of the allocated time budget of $DEFINED_TIMEOUT_MINUTES minutes ==="
	echo "=========================================================================================="
}

function log_upload_watchdog {
	SLEEP_TIME=$(($DEFINED_TIMEOUT_SECONDS-$START_LOG_UPLOAD_SECONDS_FROM_END))
	sleep $SLEEP_TIME
	echo "======================================================================================================"
	echo "=== WARNING: This E2E Run will time out in the next few minutes. Starting to upload the log output ==="
	echo "======================================================================================================"

	INDEX=0
	while true; do
		cp $OUTPUT_FILE "$OUTPUT_FILE.$INDEX"
		echo "##vso[artifact.upload containerfolder=e2e-timeout-logs;artifactname=log_upload_watchdog.output;]$OUTPUT_FILE.$INDEX"
		INDEX=$(($INDEX+1))
		sleep 300
	done
}

warning_watchdog &
log_upload_watchdog &

# ts from moreutils prepends the time to each line
( $COMMAND & PID=$! ; wait $PID ) | ts | tee $OUTPUT_FILE
TEST_EXIT_CODE=${PIPESTATUS[0]}

# properly forward exit code
exit $TEST_EXIT_CODE