| #!/usr/bin/env bash |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # Build Impala with the a variety of common build configurations and check that the build |
| # succeeds. Intended for use as a precommit test to make sure nothing got broken. |
| # |
| # Assumes that ninja and ccache are installed. |
| # |
| # Usage: build-all-flag-combinations.sh [--dryrun] |
| |
| set -euo pipefail |
| . $IMPALA_HOME/bin/report_build_error.sh |
| setup_report_build_error |
| |
| export IMPALA_MAVEN_OPTIONS="-U" |
| |
| . bin/impala-config.sh > /dev/null 2>&1 |
| |
| : ${GENERATE_M2_ARCHIVE:=false} |
| |
| # These are configurations for buildall. |
| CONFIGS=( |
| # Test gcc builds with and without -so: |
| "-skiptests -noclean" |
| "-skiptests -noclean -release" |
| "-notests -noclean -release -package" |
| "-skiptests -noclean -release -so -ninja" |
| # clang sanitizer builds: |
| "-skiptests -noclean -asan" |
| "-skiptests -noclean -tsan" |
| "-skiptests -noclean -ubsan -so -ninja" |
| # USE_APACHE_HIVE=true build: |
| "-skiptests -noclean -use_apache_components" |
| "-notests -noclean -use_apache_components -package" |
| ) |
| |
| FAILED="" |
| |
| if [[ "$GENERATE_M2_ARCHIVE" == true ]]; then |
| # The m2 archive relies on parsing the maven log to get a list of jars downloaded |
| # from particular repositories. To accurately produce the archive every time, we |
| # need to clear out the ~/.m2 directory before producing the archive. |
| rm -rf ~/.m2 |
| fi |
| |
| TMP_DIR=$(mktemp -d) |
| function onexit { |
| echo "$0: Cleaning up temporary directory" |
| rm -rf ${TMP_DIR} |
| } |
| trap onexit EXIT |
| |
| for CONFIG in "${CONFIGS[@]}"; do |
| CONFIG2=${CONFIG/-use_apache_components/} |
| if [[ "$CONFIG" != "$CONFIG2" ]]; then |
| CONFIG=$CONFIG2 |
| export USE_APACHE_COMPONENTS=true |
| else |
| export USE_APACHE_COMPONENTS=false |
| fi |
| DESCRIPTION="Options $CONFIG USE_APACHE_COMPONENTS=$USE_APACHE_COMPONENTS" |
| |
| if [[ $# == 1 && $1 == "--dryrun" ]]; then |
| echo $DESCRIPTION |
| continue |
| fi |
| |
| if ! ./bin/clean.sh; then |
| echo "Clean failed" |
| exit 1 |
| fi |
| echo "Building with OPTIONS: $DESCRIPTION" |
| if ! time -p ./buildall.sh $CONFIG; then |
| echo "Build failed: $DESCRIPTION" |
| FAILED="${FAILED}:${DESCRIPTION}" |
| fi |
| ccache -s |
| bin/jenkins/get_maven_statistics.sh logs/mvn/mvn.log |
| # Dump the current disk usage and check the usage for |
| # several heavy users of disk space. |
| df |
| du -s ~/.m2 ./be ./toolchain |
| |
| # Keep each maven log from each round of the build |
| cp logs/mvn/mvn.log "${TMP_DIR}/mvn.$(date +%s.%N).log" |
| # Append the maven log to the accumulated maven log |
| cat logs/mvn/mvn.log >> "${TMP_DIR}/mvn_accumulated.log" |
| done |
| |
| # It is useful to be able to use "mvn versions:set" to modify the jar |
| # versions produced by Impala. This tests that Impala continues to build |
| # after running that command. |
| NEW_MAVEN_VERSION=4.0.0-$(date +%Y%m%d) |
| DESCRIPTION="Options: mvn versions:set -DnewVersion=${NEW_MAVEN_VERSION}" |
| |
| if [[ $# == 1 && $1 == "--dryrun" ]]; then |
| echo $DESCRIPTION |
| else |
| # Note: this command modifies files in the git checkout |
| pushd bin |
| sed "s#export IMPALA_VERSION=.*#export IMPALA_VERSION=${NEW_MAVEN_VERSION}#g"\ |
| -i impala-config.sh |
| popd |
| pushd java |
| mvn org.codehaus.mojo:versions-maven-plugin:2.13.0:set -DnewVersion=${NEW_MAVEN_VERSION} |
| popd |
| |
| if ! ./bin/clean.sh; then |
| echo "Clean failed" |
| exit 1 |
| fi |
| |
| echo "Building with OPTIONS: $DESCRIPTION" |
| # Since this is only testing the version replacements for Java, this uses -notests to |
| # skip building the backend tests. Because everything is already in ccache, building |
| # backend tests does many link invocations simultaneously and the system can run out of |
| # memory. |
| if ! time -p ./buildall.sh -notests -noclean ; then |
| echo "Build failed: $DESCRIPTION" |
| FAILED="${FAILED}:${DESCRIPTION}" |
| fi |
| |
| if ! git reset --hard HEAD ; then |
| echo "Failed to reset the files changed by mvn versions:set" |
| fi |
| |
| ccache -s |
| bin/jenkins/get_maven_statistics.sh logs/mvn/mvn.log |
| # Dump the current disk usage and check the usage for |
| # several heavy users of disk space. |
| df |
| du -s ~/.m2 ./be ./toolchain |
| |
| # Keep each maven log from each round of the build |
| cp logs/mvn/mvn.log "${TMP_DIR}/mvn.$(date +%s.%N).log" |
| # Append the maven log to the accumulated maven log |
| cat logs/mvn/mvn.log >> "${TMP_DIR}/mvn_accumulated.log" |
| fi |
| |
| # To avoid diskspace issues, clean up the Impala build directory |
| # before creating the archive. |
| rm -rf be/build |
| |
| # Restore the maven logs (these don't interfere with existing mvn.log) |
| # These files may not exist if this is doing a dryrun. |
| if ls ${TMP_DIR}/mvn* > /dev/null 2>&1 ; then |
| cp ${TMP_DIR}/mvn* logs/mvn |
| fi |
| |
| if [[ "$FAILED" != "" ]] |
| then |
| echo "The following builds failed:" |
| echo "$FAILED" |
| exit 1 |
| fi |
| |
| if [[ "$GENERATE_M2_ARCHIVE" == true ]]; then |
| echo "Generating m2 archive." |
| |
| # Print current diskspace |
| df |
| |
| # Make a tarball of the .m2 directory |
| bin/jenkins/archive_m2_directory.sh logs/mvn/mvn_accumulated.log logs/m2_archive.tar.gz |
| fi |
| |
| # Note: The exit callback handles cleanup of the temp directory. |