| #!/usr/bin/env bash |
| |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| # Go to the Spark project root directory |
| FWDIR="$(cd `dirname $0`/..; pwd)" |
| cd "$FWDIR" |
| |
| if [ -n "$AMPLAB_JENKINS_BUILD_PROFILE" ]; then |
| if [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop1.0" ]; then |
| export SBT_MAVEN_PROFILES_ARGS="-Dhadoop.version=1.0.4" |
| elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.0" ]; then |
| export SBT_MAVEN_PROFILES_ARGS="-Dhadoop.version=2.0.0-mr1-cdh4.1.1" |
| elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.2" ]; then |
| export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Dhadoop.version=2.2.0" |
| elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.3" ]; then |
| export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0" |
| fi |
| fi |
| |
| if [ -z "$SBT_MAVEN_PROFILES_ARGS" ]; then |
| export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0" |
| fi |
| |
| export SBT_MAVEN_PROFILES_ARGS="$SBT_MAVEN_PROFILES_ARGS -Pkinesis-asl" |
| |
| echo "SBT_MAVEN_PROFILES_ARGS=\"$SBT_MAVEN_PROFILES_ARGS\"" |
| |
| # Remove work directory |
| rm -rf ./work |
| |
| if test -x "$JAVA_HOME/bin/java"; then |
| declare java_cmd="$JAVA_HOME/bin/java" |
| else |
| declare java_cmd=java |
| fi |
| JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') |
| [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8." |
| |
| # Partial solution for SPARK-1455. Only run Hive tests if there are sql changes. |
| if [ -n "$AMPLAB_JENKINS" ]; then |
| git fetch origin master:master |
| diffs=`git diff --name-only master | grep "^\(sql/\)\|\(bin/spark-sql\)\|\(sbin/start-thriftserver.sh\)"` |
| if [ -n "$diffs" ]; then |
| echo "Detected changes in SQL. Will run Hive test suite." |
| _RUN_SQL_TESTS=true |
| fi |
| fi |
| |
| # Fail fast |
| set -e |
| set -o pipefail |
| |
| echo "" |
| echo "=========================================================================" |
| echo "Running Apache RAT checks" |
| echo "=========================================================================" |
| dev/check-license |
| |
| echo "" |
| echo "=========================================================================" |
| echo "Running Scala style checks" |
| echo "=========================================================================" |
| dev/lint-scala |
| |
| echo "" |
| echo "=========================================================================" |
| echo "Running Python style checks" |
| echo "=========================================================================" |
| dev/lint-python |
| |
| echo "" |
| echo "=========================================================================" |
| echo "Running Spark unit tests" |
| echo "=========================================================================" |
| |
| # Build Spark; we always build with Hive because the PySpark SparkSQL tests need it. |
| # echo "q" is needed because sbt on encountering a build file with failure |
| # (either resolution or compilation) prompts the user for input either q, r, |
| # etc to quit or retry. This echo is there to make it not block. |
| BUILD_MVN_PROFILE_ARGS="$SBT_MAVEN_PROFILES_ARGS -Phive -Phive-thriftserver " |
| echo -e "q\n" | sbt/sbt $BUILD_MVN_PROFILE_ARGS clean package assembly/assembly | \ |
| grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including" |
| |
| # If the Spark SQL tests are enabled, run the tests with the Hive profiles enabled: |
| if [ -n "$_RUN_SQL_TESTS" ]; then |
| SBT_MAVEN_PROFILES_ARGS="$SBT_MAVEN_PROFILES_ARGS -Phive" |
| fi |
| # echo "q" is needed because sbt on encountering a build file with failure |
| # (either resolution or compilation) prompts the user for input either q, r, |
| # etc to quit or retry. This echo is there to make it not block. |
| echo -e "q\n" | sbt/sbt $SBT_MAVEN_PROFILES_ARGS test | \ |
| grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including" |
| |
| echo "" |
| echo "=========================================================================" |
| echo "Running PySpark tests" |
| echo "=========================================================================" |
| ./python/run-tests |
| |
| echo "" |
| echo "=========================================================================" |
| echo "Detecting binary incompatibilites with MiMa" |
| echo "=========================================================================" |
| dev/mima |