| #!/usr/bin/env bash |
| ################################################################################ |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| ################################################################################ |
| |
| HERE="`dirname \"$0\"`" # relative |
| HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized |
| if [ -z "$HERE" ] ; then |
| # error; for some reason, the path is not accessible |
| # to the script (e.g. permissions re-evaled after suid) |
| exit 1 # fail |
| fi |
| |
| ARTIFACTS_DIR="${HERE}/artifacts" |
| |
| mkdir -p $ARTIFACTS_DIR || { echo "FAILURE: cannot create log directory '${ARTIFACTS_DIR}'." ; exit 1; } |
| |
| echo "Build for commit ${TRAVIS_COMMIT} of ${TRAVIS_REPO_SLUG} [build ID: ${TRAVIS_BUILD_ID}, job number: $TRAVIS_JOB_NUMBER]." | tee "${ARTIFACTS_DIR}/build_info" |
| |
| # ============================================================================= |
| # CONFIG |
| # ============================================================================= |
| |
| # Number of seconds w/o output before printing a stack trace and killing $MVN |
| MAX_NO_OUTPUT=${1:-300} |
| |
| # Number of seconds to sleep before checking the output again |
| SLEEP_TIME=20 |
| |
| LOG4J_PROPERTIES=${HERE}/log4j-travis.properties |
| |
| # Maven command to run. We set the forkCount manually, because otherwise Maven sees too many cores |
| # on the Travis VMs. |
| MVN="mvn -Dflink.forkCount=2 -B $PROFILE -Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES clean install" |
| |
| MVN_PID="${ARTIFACTS_DIR}/watchdog.mvn.pid" |
| MVN_EXIT="${ARTIFACTS_DIR}/watchdog.mvn.exit" |
| MVN_OUT="${ARTIFACTS_DIR}/mvn.out" |
| |
| TRACE_OUT="${ARTIFACTS_DIR}/jps-traces.out" |
| |
| # E.g. travis-artifacts/apache/flink/1595/1595.1 |
| UPLOAD_TARGET_PATH="travis-artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BUILD_NUMBER}/" |
| # These variables are stored as secure variables in '.travis.yml', which are generated per repo via |
| # the travis command line tool. |
| UPLOAD_BUCKET=$ARTIFACTS_AWS_BUCKET |
| UPLOAD_ACCESS_KEY=$ARTIFACTS_AWS_ACCESS_KEY |
| UPLOAD_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY |
| |
| ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz |
| |
| # ============================================================================= |
| # FUNCTIONS |
| # ============================================================================= |
| |
| upload_artifacts_s3() { |
| echo "PRODUCED build artifacts." |
| |
| ls $ARTIFACTS_DIR |
| |
| if [ -n "$UPLOAD_BUCKET" ] && [ -n "$UPLOAD_ACCESS_KEY" ] && [ -n "$UPLOAD_SECRET_KEY" ]; then |
| echo "COMPRESSING build artifacts." |
| |
| cd $ARTIFACTS_DIR |
| tar -zcvf $ARTIFACTS_FILE * |
| |
| # Install artifacts tool |
| curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash |
| |
| PATH=$HOME/bin/artifacts:$HOME/bin:$PATH |
| |
| echo "UPLOADING build artifacts." |
| |
| # Upload everything in $ARTIFACTS_DIR. Use relative path, otherwise the upload tool |
| # re-creates the whole directory structure from root. |
| artifacts upload --bucket $UPLOAD_BUCKET --key $UPLOAD_ACCESS_KEY --secret $UPLOAD_SECRET_KEY --target-paths $UPLOAD_TARGET_PATH $ARTIFACTS_FILE |
| fi |
| } |
| |
| print_stacktraces () { |
| echo "==============================================================================" |
| echo "The following Java processes are running (JPS)" |
| echo "==============================================================================" |
| |
| jps |
| |
| local pids=( $(jps | awk '{print $1}') ) |
| |
| for pid in "${pids[@]}"; do |
| echo "==============================================================================" |
| echo "Printing stack trace of Java process ${pid}" |
| echo "==============================================================================" |
| |
| jstack $pid |
| done |
| } |
| |
| # locate YARN logs and put them into artifacts directory |
| put_yarn_logs_to_artifacts() { |
| # Make sure to be in project root |
| cd $HERE/../ |
| for file in `find ./flink-yarn-tests/target/flink-yarn-tests* -type f -name '*.log'`; do |
| TARGET_FILE=`echo "$file" | grep -Eo "container_[0-9_]+/(.*).log"` |
| TARGET_DIR=`dirname "$TARGET_FILE"` |
| mkdir -p "$ARTIFACTS_DIR/yarn-tests/$TARGET_DIR" |
| cp $file "$ARTIFACTS_DIR/yarn-tests/$TARGET_FILE" |
| done |
| } |
| |
| mod_time () { |
| if [[ `uname` == 'Darwin' ]]; then |
| eval $(stat -s $MVN_OUT) |
| echo $st_mtime |
| else |
| echo `stat -c "%Y" $MVN_OUT` |
| fi |
| } |
| |
| the_time() { |
| echo `date +%s` |
| } |
| |
| watchdog () { |
| touch $MVN_OUT |
| |
| while true; do |
| sleep $SLEEP_TIME |
| |
| time_diff=$((`the_time` - `mod_time`)) |
| |
| if [ $time_diff -ge $MAX_NO_OUTPUT ]; then |
| echo "==============================================================================" |
| echo "Maven produced no output for ${MAX_NO_OUTPUT} seconds." |
| echo "==============================================================================" |
| |
| print_stacktraces | tee $TRACE_OUT |
| |
| kill $(<$MVN_PID) |
| |
| exit 1 |
| fi |
| done |
| } |
| |
| # Check the final fat jar for illegal artifacts |
| check_shaded_artifacts() { |
| jar tf build-target/lib/flink-dist-*.jar > allClasses |
| ASM=`cat allClasses | grep '^org/objectweb/asm/' | wc -l` |
| if [ $ASM != "0" ]; then |
| echo "==============================================================================" |
| echo "Detected $ASM asm dependencies in fat jar" |
| echo "==============================================================================" |
| exit 1 |
| fi |
| |
| GUAVA=`cat allClasses | grep '^com/google/common' | wc -l` |
| if [ $GUAVA != "0" ]; then |
| echo "==============================================================================" |
| echo "Detected $GUAVA guava dependencies in fat jar" |
| echo "==============================================================================" |
| exit 1 |
| fi |
| |
| } |
| |
| # ============================================================================= |
| # WATCHDOG |
| # ============================================================================= |
| |
| # Start watching $MVN_OUT |
| watchdog & |
| |
| WD_PID=$! |
| |
| echo "STARTED watchdog (${WD_PID})." |
| |
| # Make sure to be in project root |
| cd $HERE/../ |
| |
| echo "RUNNING '${MVN}'." |
| |
| # Run $MVN and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to |
| # allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains |
| # the exit code. This is important for Travis' build life-cycle (success/failure). |
| ( $MVN & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT |
| |
| echo "Trying to KILL watchdog (${WD_PID})." |
| |
| # Make sure to kill the watchdog in any case after $MVN has completed |
| ( kill $WD_PID 2>&1 ) > /dev/null |
| |
| EXIT_CODE=$(<$MVN_EXIT) |
| |
| echo "MVN exited with EXIT CODE: ${EXIT_CODE}." |
| |
| rm $MVN_PID |
| rm $MVN_EXIT |
| |
| check_shaded_artifacts |
| |
| put_yarn_logs_to_artifacts |
| |
| upload_artifacts_s3 |
| |
| # Check the number of files in the uber jar and fail the build if there are too many files (see: FLINK-1637) |
| |
| # since we are in flink/tools/artifacts |
| # we are going back to |
| cd ../../ |
| |
| |
| UBERJAR=`find . | grep flink-dist | grep jar | head -n 1` |
| if [ -z "$UBERJAR" ] ; then |
| echo "Uberjar not found. Assuming failed build"; |
| else |
| jar tf $UBERJAR | wc -l > num_files_in_uberjar |
| NUM_FILES_IN_UBERJAR=`cat num_files_in_uberjar` |
| echo "Files in uberjar: $NUM_FILES_IN_UBERJAR. Uberjar: $UBERJAR" |
| if [ "$NUM_FILES_IN_UBERJAR" -ge "65536" ] ; then |
| echo "WARN: The number of files in the uberjar ($NUM_FILES_IN_UBERJAR) exceeds the maximum number of possible files for Java 6 (65536)" |
| fi |
| fi |
| |
| # Exit code for Travis build success/failure |
| exit $EXIT_CODE |