| #!/usr/bin/env bash |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # |
| # SHELLDOC-IGNORE |
| # |
| # Override these to match Apache Hadoop's requirements |
| personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" |
| |
| ## @description Globals specific to this personality |
| ## @audience private |
| ## @stability evolving |
| function personality_globals |
| { |
| # shellcheck disable=SC2034 |
| BUILDTOOL=maven |
| #shellcheck disable=SC2034 |
| PATCH_BRANCH_DEFAULT=trunk |
| #shellcheck disable=SC2034 |
| PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute" |
| #shellcheck disable=SC2034 |
| JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' |
| #shellcheck disable=SC2034 |
| GITHUB_REPO_DEFAULT="apache/hadoop" |
| |
| HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)} |
| if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then |
| HADOOP_HOMEBREW_DIR=/usr/local |
| fi |
| } |
| |
| function personality_parse_args |
| { |
| declare i |
| |
| for i in "$@"; do |
| case ${i} in |
| --hadoop-isal-prefix=*) |
| delete_parameter "${i}" |
| ISAL_HOME=${i#*=} |
| ;; |
| --hadoop-openssl-prefix=*) |
| delete_parameter "${i}" |
| OPENSSL_HOME=${i#*=} |
| ;; |
| --hadoop-snappy-prefix=*) |
| delete_parameter "${i}" |
| SNAPPY_HOME=${i#*=} |
| ;; |
| esac |
| done |
| } |
| |
| ## @description Calculate the actual module ordering |
| ## @audience private |
| ## @stability evolving |
| ## @param ordering |
| function hadoop_order |
| { |
| declare ordering=$1 |
| declare hadoopm |
| |
| if [[ ${ordering} = normal ]]; then |
| hadoopm="${CHANGED_MODULES[*]}" |
| elif [[ ${ordering} = union ]]; then |
| hadoopm="${CHANGED_UNION_MODULES}" |
| elif [[ ${ordering} = mvnsrc ]]; then |
| hadoopm="${MAVEN_SRC_MODULES[*]}" |
| elif [[ ${ordering} = mvnsrctest ]]; then |
| hadoopm="${MAVEN_SRCTEST_MODULES[*]}" |
| else |
| hadoopm="${ordering}" |
| fi |
| echo "${hadoopm}" |
| } |
| |
| ## @description Determine if it is safe to run parallel tests |
| ## @audience private |
| ## @stability evolving |
| ## @param ordering |
| function hadoop_test_parallel |
| { |
| if [[ -f "${BASEDIR}/pom.xml" ]]; then |
| HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \ |
| | head -1 \ |
| | "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \ |
| | cut -f1 -d- ) |
| export HADOOP_VERSION |
| else |
| return 1 |
| fi |
| |
| hmajor=${HADOOP_VERSION%%\.*} |
| hmajorminor=${HADOOP_VERSION%\.*} |
| hminor=${hmajorminor##*\.} |
| # ... and just for reference |
| #hmicro=${HADOOP_VERSION##*\.} |
| |
| # Apache Hadoop v2.8.0 was the first one to really |
| # get working parallel unit tests |
| if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then |
| return 1 |
| fi |
| |
| return 0 |
| } |
| |
| ## @description Install extra modules for unit tests |
| ## @audience private |
| ## @stability evolving |
| ## @param ordering |
| function hadoop_unittest_prereqs |
| { |
| declare input=$1 |
| declare mods |
| declare need_common=0 |
| declare building_common=0 |
| declare module |
| declare flags |
| declare fn |
| |
| # prior to running unit tests, hdfs needs libhadoop.so built |
| # if we're building root, then this extra work is moot |
| |
| #shellcheck disable=SC2086 |
| mods=$(hadoop_order ${input}) |
| |
| for module in ${mods}; do |
| if [[ ${module} = hadoop-hdfs-project* ]]; then |
| need_common=1 |
| elif [[ ${module} = hadoop-common-project/hadoop-common |
| || ${module} = hadoop-common-project ]]; then |
| building_common=1 |
| elif [[ ${module} = . ]]; then |
| return |
| fi |
| done |
| |
| # Windows builds *ALWAYS* need hadoop-common compiled |
| case ${OSTYPE} in |
| Windows_NT|CYGWIN*|MINGW*|MSYS*) |
| need_common=1 |
| ;; |
| esac |
| |
| if [[ ${need_common} -eq 1 |
| && ${building_common} -eq 0 ]]; then |
| echo "unit test pre-reqs:" |
| module="hadoop-common-project/hadoop-common" |
| fn=$(module_file_fragment "${module}") |
| flags="$(hadoop_native_flags) $(yarn_ui2_flag)" |
| pushd "${BASEDIR}/${module}" >/dev/null || return 1 |
| # shellcheck disable=SC2086 |
| echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \ |
| "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags} |
| popd >/dev/null || return 1 |
| fi |
| } |
| |
| ## @description Calculate the flags/settings for yarn-ui v2 build |
| ## @description based upon the OS |
| ## @audience private |
| ## @stability evolving |
| function yarn_ui2_flag |
| { |
| |
| if [[ ${BUILD_NATIVE} != true ]]; then |
| return |
| fi |
| |
| # Now it only tested on Linux/OSX, don't enable the profile on |
| # windows until it get verified |
| case ${OSTYPE} in |
| Linux) |
| # shellcheck disable=SC2086 |
| echo -Pyarn-ui |
| ;; |
| Darwin) |
| echo -Pyarn-ui |
| ;; |
| *) |
| # Do nothing |
| ;; |
| esac |
| } |
| |
| ## @description Calculate the flags/settings for native code |
| ## @description based upon the OS |
| ## @audience private |
| ## @stability evolving |
| function hadoop_native_flags |
| { |
| if [[ ${BUILD_NATIVE} != true ]]; then |
| return |
| fi |
| |
| declare -a args |
| |
| # Based upon HADOOP-11937 |
| # |
| # Some notes: |
| # |
| # - getting fuse to compile on anything but Linux |
| # is always tricky. |
| # - Darwin assumes homebrew is in use. |
| # - HADOOP-12027 required for bzip2 on OS X. |
| # - bzip2 is broken in lots of places |
| # (the shared library is considered experimental) |
| # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2 |
| # |
| |
| args=("-Drequire.test.libhadoop") |
| |
| if [[ -d "${ISAL_HOME}/include" ]]; then |
| args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}") |
| fi |
| |
| if [[ -d "${OPENSSL_HOME}/include" ]]; then |
| args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}") |
| elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then |
| args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/") |
| fi |
| |
| if [[ -d "${SNAPPY_HOME}/include" ]]; then |
| args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}") |
| elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then |
| args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy") |
| fi |
| |
| case ${OSTYPE} in |
| Linux) |
| # shellcheck disable=SC2086 |
| echo \ |
| -Pnative \ |
| -Drequire.fuse \ |
| -Drequire.openssl \ |
| -Drequire.snappy \ |
| -Drequire.valgrind \ |
| -Drequire.zstd \ |
| "${args[@]}" |
| ;; |
| Darwin) |
| echo \ |
| "${args[@]}" \ |
| -Pnative \ |
| -Drequire.snappy \ |
| -Drequire.openssl |
| ;; |
| Windows_NT|CYGWIN*|MINGW*|MSYS*) |
| echo \ |
| "${args[@]}" \ |
| -Drequire.snappy -Drequire.openssl -Pnative-win |
| ;; |
| *) |
| echo \ |
| "${args[@]}" |
| ;; |
| esac |
| } |
| |
| ## @description Queue up modules for this personality |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| ## @param testtype |
| function personality_modules |
| { |
| declare repostatus=$1 |
| declare testtype=$2 |
| declare extra="" |
| declare ordering="normal" |
| declare needflags=false |
| declare foundbats=false |
| declare flags |
| declare fn |
| declare i |
| declare hadoopm |
| |
| yetus_debug "Personality: ${repostatus} ${testtype}" |
| |
| clear_personality_queue |
| |
| case ${testtype} in |
| asflicense) |
| # this is very fast and provides the full path if we do it from |
| # the root of the source |
| personality_enqueue_module . |
| return |
| ;; |
| checkstyle) |
| ordering="union" |
| extra="-DskipTests" |
| ;; |
| compile) |
| ordering="union" |
| extra="-DskipTests" |
| needflags=true |
| |
| # if something in common changed, we build the whole world |
| if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then |
| yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . " |
| ordering="." |
| fi |
| ;; |
| distclean) |
| ordering="." |
| extra="-DskipTests" |
| ;; |
| javadoc) |
| if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then |
| ordering=. |
| fi |
| |
| if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then |
| echo "javadoc pre-reqs:" |
| for i in hadoop-project \ |
| hadoop-common-project/hadoop-annotations; do |
| fn=$(module_file_fragment "${i}") |
| pushd "${BASEDIR}/${i}" >/dev/null || return 1 |
| echo "cd ${i}" |
| echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \ |
| "${MAVEN}" "${MAVEN_ARGS[@]}" install |
| popd >/dev/null || return 1 |
| done |
| fi |
| extra="-Pdocs -DskipTests" |
| ;; |
| mvneclipse) |
| if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then |
| ordering=. |
| fi |
| ;; |
| mvninstall) |
| extra="-DskipTests" |
| if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then |
| ordering=. |
| fi |
| ;; |
| mvnsite) |
| if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then |
| ordering=. |
| fi |
| ;; |
| unit) |
| if [[ "${BUILDMODE}" = full ]]; then |
| ordering=mvnsrc |
| elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then |
| ordering=. |
| fi |
| |
| if [[ ${TEST_PARALLEL} = "true" ]] ; then |
| if hadoop_test_parallel; then |
| extra="-Pparallel-tests" |
| if [[ -n ${TEST_THREADS:-} ]]; then |
| extra="${extra} -DtestsThreadCount=${TEST_THREADS}" |
| fi |
| fi |
| fi |
| needflags=true |
| hadoop_unittest_prereqs "${ordering}" |
| |
| if ! verify_needed_test javac; then |
| yetus_debug "hadoop: javac not requested" |
| if ! verify_needed_test native; then |
| yetus_debug "hadoop: native not requested" |
| yetus_debug "hadoop: adding -DskipTests to unit test" |
| extra="-DskipTests" |
| fi |
| fi |
| |
| for i in "${CHANGED_FILES[@]}"; do |
| if [[ "${i}" =~ \.bats ]]; then |
| foundbats=true |
| fi |
| done |
| |
| if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then |
| yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile" |
| extra="${extra} -P!shelltest" |
| else |
| extra="${extra} -Pshelltest" |
| fi |
| ;; |
| *) |
| extra="-DskipTests" |
| ;; |
| esac |
| |
| if [[ ${needflags} = true ]]; then |
| flags="$(hadoop_native_flags) $(yarn_ui2_flag)" |
| extra="${extra} ${flags}" |
| fi |
| |
| extra="-Ptest-patch ${extra}" |
| for module in $(hadoop_order ${ordering}); do |
| # shellcheck disable=SC2086 |
| personality_enqueue_module ${module} ${extra} |
| done |
| } |
| |
| ## @description Add tests based upon personality needs |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function personality_file_tests |
| { |
| declare filename=$1 |
| |
| yetus_debug "Using Hadoop-specific personality_file_tests" |
| |
| if [[ ${filename} =~ src/main/webapp ]]; then |
| yetus_debug "tests/webapp: ${filename}" |
| add_test shadedclient |
| elif [[ ${filename} =~ \.sh |
| || ${filename} =~ \.cmd |
| || ${filename} =~ src/scripts |
| || ${filename} =~ src/test/scripts |
| || ${filename} =~ src/main/bin |
| || ${filename} =~ shellprofile\.d |
| || ${filename} =~ src/main/conf |
| ]]; then |
| yetus_debug "tests/shell: ${filename}" |
| add_test mvnsite |
| add_test unit |
| elif [[ ${filename} =~ \.md$ |
| || ${filename} =~ \.md\.vm$ |
| || ${filename} =~ src/site |
| ]]; then |
| yetus_debug "tests/site: ${filename}" |
| add_test mvnsite |
| elif [[ ${filename} =~ \.c$ |
| || ${filename} =~ \.cc$ |
| || ${filename} =~ \.h$ |
| || ${filename} =~ \.hh$ |
| || ${filename} =~ \.proto$ |
| || ${filename} =~ \.cmake$ |
| || ${filename} =~ CMakeLists.txt |
| ]]; then |
| yetus_debug "tests/units: ${filename}" |
| add_test compile |
| add_test cc |
| add_test mvnsite |
| add_test javac |
| add_test unit |
| elif [[ ${filename} =~ build.xml$ |
| || ${filename} =~ pom.xml$ |
| || ${filename} =~ \.java$ |
| || ${filename} =~ src/main |
| ]]; then |
| yetus_debug "tests/javadoc+units: ${filename}" |
| add_test compile |
| add_test javac |
| add_test javadoc |
| add_test mvninstall |
| add_test mvnsite |
| add_test unit |
| add_test shadedclient |
| fi |
| |
| # if we change anything in here, e.g. the test scripts |
| # then run the client artifact tests |
| if [[ ${filename} =~ hadoop-client-modules ]]; then |
| add_test shadedclient |
| fi |
| |
| if [[ ${filename} =~ src/test ]]; then |
| yetus_debug "tests: src/test" |
| add_test unit |
| fi |
| |
| if [[ ${filename} =~ \.java$ ]]; then |
| add_test findbugs |
| fi |
| } |
| |
| ## @description Image to print on success |
| ## @audience private |
| ## @stability evolving |
| function hadoop_console_success |
| { |
| printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; |
| printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; |
| printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; |
| printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; |
| printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; |
| printf "ICB8bXwgICB8bXwgIAo" |
| } |
| |
| ################################################### |
| # Hadoop project specific check of IT for shaded artifacts |
| |
| add_test_type shadedclient |
| |
| ## @description check for test modules and add test/plugins as needed |
| ## @audience private |
| ## @stability evolving |
| function shadedclient_initialize |
| { |
| maven_add_install shadedclient |
| } |
| |
| ## @description build client facing shaded artifacts and test them |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| function shadedclient_rebuild |
| { |
| declare repostatus=$1 |
| declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt" |
| declare module |
| declare -a modules=() |
| |
| if [[ ${OSTYPE} = Windows_NT || |
| ${OSTYPE} =~ ^CYGWIN.* || |
| ${OSTYPE} =~ ^MINGW32.* || |
| ${OSTYPE} =~ ^MSYS.* ]]; then |
| echo "hadoop personality: building on windows, skipping check of client artifacts." |
| return 0 |
| fi |
| |
| yetus_debug "hadoop personality: seeing if we need the test of client artifacts." |
| for module in hadoop-client-modules/hadoop-client-check-invariants \ |
| hadoop-client-modules/hadoop-client-check-test-invariants \ |
| hadoop-client-modules/hadoop-client-integration-tests; do |
| if [ -d "${module}" ]; then |
| yetus_debug "hadoop personality: test module '${module}' is present." |
| modules+=(-pl "${module}") |
| fi |
| done |
| if [ ${#modules[@]} -eq 0 ]; then |
| echo "hadoop personality: no test modules present, skipping check of client artifacts." |
| return 0 |
| fi |
| |
| big_console_header "Checking client artifacts on ${repostatus}" |
| |
| echo_and_redirect "${logfile}" \ |
| "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ |
| "${modules[@]}" \ |
| -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true |
| |
| count=$("${GREP}" -c '\[ERROR\]' "${logfile}") |
| if [[ ${count} -gt 0 ]]; then |
| add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts." |
| return 1 |
| fi |
| |
| add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts." |
| return 0 |
| } |
