dev/phoenix-connectors-personality.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Based on hbase-personality.sh of the HBase project
#
# You'll need a local installation of
# [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.12.0/#yetus-precommit)
# to use this personality.
#
# Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
# build from it, or use the convenience binaries provided on that download page.
#
# To run against, e.g. PHOENIX-5032 you'd then do
# ```bash
# test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
# ```
#
# pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
# directory / branch in order to have things match what the issue patch requests.

personality_plugins "all"

if ! declare -f "yetus_info" >/dev/null; then

  function yetus_info
  {
    echo "[$(date) INFO]: $*" 1>&2
  }

fi

# work around yetus overwriting JAVA_HOME from our docker image
#function docker_do_env_adds
#{
#  declare k
#  for k in "${DOCKER_EXTRAENVS[@]}"; do
#    if [[ "JAVA_HOME" == "${k}" ]]; then
#      if [ -n "${JAVA_HOME}" ]; then
#        DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
#      fi
#    else
#      DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
#    fi
#  done
#}


## @description  Globals specific to this personality
## @audience     private
## @stability    evolving
function personality_globals
{
  BUILDTOOL=maven
  #shellcheck disable=SC2034
  PROJECT_NAME=phoenix-connectors
  #shellcheck disable=SC2034
  PATCH_BRANCH_DEFAULT=master
  #shellcheck disable=SC2034
  JIRA_ISSUE_RE='^PHOENIX-[0-9]+$'
  #shellcheck disable=SC2034
  GITHUB_REPO="apache/phoenix-connectors"

  # TODO use PATCH_BRANCH to select jdk versions to use.

  # ASF Jenkins workers run up to two jobs per Agent. Docker doesn't do anything with nproc,
  # any setting is simply set on the docker daemon user, and shared between containers.
  # Thus, there is no way to protect containers from fork-bombing each other.
  # Set nprocs higher than than the combined process count of two jobs.
  # see https://docs.docker.com/engine/reference/commandline/run/#set-ulimits-in-container-ulimit
  # Note that this won't stop a container started after us from resetting the limit to a
  # lower value, but should help if we are started later.
  #shellcheck disable=SC2034
  PROC_LIMIT=30000

  # Set docker container to run with 20g. Default is 4g in yetus.
  # See HBASE-19902 for how we arrived at 20g.
  # TODO Doesn't seem to have effect in Yetus 0.12, set in cli instead
  #shellcheck disable=SC2034
  DOCKERMEMLIMIT=20g
}

## @description  Parse extra arguments required by personalities, if any.
## @audience     private
## @stability    evolving
function personality_parse_args
{
  declare i

  for i in "$@"; do
    case ${i} in
      --exclude-tests-url=*)
        delete_parameter "${i}"
        EXCLUDE_TESTS_URL=${i#*=}
      ;;
      --include-tests-url=*)
        delete_parameter "${i}"
        INCLUDE_TESTS_URL=${i#*=}
      ;;
      --hbase-profile=*)
        delete_parameter "${i}"
        HBASE_PROFILE=${i#*=}
      ;;
      --skip-errorprone)
        delete_parameter "${i}"
        SKIP_ERRORPRONE=true
      ;;
    esac
  done
}

## @description  Queue up modules for this personality
## @audience     private
## @stability    evolving
## @param        repostatus
## @param        testtype
function personality_modules
{
  local repostatus=$1
  local testtype=$2
  local extra=""
  local jdk8module=""
  local MODULES=("${CHANGED_MODULES[@]}")

  yetus_info "Personality: ${repostatus} ${testtype}"

  clear_personality_queue

  # TODO: test with threads>0, once the tests are stable
  extra="--threads=1 -DPhoenixPatchProcess"
  if [[ "${PATCH_BRANCH}" = 4* ]]; then
    extra="${extra} -Dhttps.protocols=TLSv1.2"
  fi

  # If we have HBASE_PROFILE specified pass along
  # the hadoop.profile system property.
  if [[ -n "${HBASE_PROFILE}" ]] ; then
    extra="${extra} -Dhbase.profile=${HBASE_PROFILE}"
  fi

  # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
  # full source code instead. In this case, do full compiles, tests, etc instead of per
  # module.
  # Used in nightly runs.
  # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need to run individual
  # modules if root is included. HBASE-18505
  if [[ "${BUILDMODE}" == "full" ]] || \
     { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[ "${testtype}" == checkstyle ]]; } && \
     [[ "${MODULES[*]}" =~ \. ]]; }; then
    MODULES=(.)
  fi

  # If the checkstyle configs change, check everything.
  if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
    MODULES=(.)
  fi

  if [[ ${testtype} == mvninstall ]]; then
    # shellcheck disable=SC2086
    personality_enqueue_module . ${extra}
    return
  fi

  if [[ ${testtype} == spotbugs ]]; then
    # Run spotbugs on each module individually to diff pre-patch and post-patch results and
    # report new warnings for changed modules only.
    # For some reason, spotbugs on root is not working, but running on individual modules is
    # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
    for module in "${CHANGED_MODULES[@]}"; do
      # skip spotbugs on any module that lacks content in `src/main/java`
      if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath '*/src/main/java/*' \
          -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
        yetus_debug "no java files found under ${module}/src/main/java. skipping."
        continue
      else
        # shellcheck disable=SC2086
        personality_enqueue_module ${module} ${extra}
      fi
    done
    return
  fi

  if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
      [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
    extra="${extra} -PerrorProne"
  fi

  # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
  # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
  # tests respectively.
  if [[ ${testtype} == unit ]]; then
    local tests_arg=""
    get_include_exclude_tests_arg tests_arg
    #Phoenix projects traditially runs the full IT suite from Precommit
    #keep it that way to ease transition
    extra="verify ${extra} ${tests_arg}"

    # Inject the jenkins build-id for our surefire invocations
    # Used by zombie detection stuff, even though we're not including that yet.
    if [ -n "${BUILD_ID}" ]; then
      extra="${extra} -Dbuild.id=${BUILD_ID}"
    fi

  fi

  for module in "${MODULES[@]}"; do
    # shellcheck disable=SC2086
    personality_enqueue_module ${module} ${extra}
  done
}

## @description places where we override the built in assumptions about what tests to run
## @audience    private
## @stability   evolving
## @param       filename of changed file
function personality_file_tests
{
  local filename=$1
  yetus_debug "Phoenix connectors specific personality_file_tests"
  # If we change checkstyle configs, run checkstyle
  if [[ ${filename} =~ checkstyle.*\.xml ]]; then
    yetus_debug "tests/checkstyle: ${filename}"
    add_test checkstyle
  fi
  # fallback to checking which tests based on what yetus would do by default
  if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
    "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
  elif declare -f builtin_personality_file_tests >/dev/null; then
    builtin_personality_file_tests "${filename}"
fi
}

## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
#                tests and sets given variable to arguments to be passes to maven command.
## @audience     private
## @stability    evolving
## @param        name of variable to set with maven arguments
function get_include_exclude_tests_arg
{
  #Phoenix projects doesn't support this yet, but should
  return
  local  __resultvar=$1
  yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
  yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
  if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
      if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
        excludes=$(cat excludes)
        yetus_debug "excludes=${excludes}"
        if [[ -n "${excludes}" ]]; then
          eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
        fi
        rm excludes
      else
        yetus_error "Wget error $? in fetching excludes file from url" \
             "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
      fi
  elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
      if wget "$INCLUDE_TESTS_URL" -O "includes"; then
        includes=$(cat includes)
        yetus_debug "includes=${includes}"
        if [[ -n "${includes}" ]]; then
          eval "${__resultvar}='-Dtest=${includes}'"
        fi
        rm includes
      else
        yetus_error "Wget error $? in fetching includes file from url" \
             "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
      fi
  else
    # Use branch specific exclude list when EXCLUDE_TESTS_URL and INCLUDE_TESTS_URL are empty
    FLAKY_URL="https://ci-hadoop.apache.org/job/HBase/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/excludes/"
    if wget "${FLAKY_URL}" -O "excludes"; then
      excludes=$(cat excludes)
        yetus_debug "excludes=${excludes}"
        if [[ -n "${excludes}" ]]; then
          eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
        fi
        rm excludes
      else
        yetus_error "Wget error $? in fetching excludes file from url" \
             "${FLAKY_URL}. Ignoring and proceeding."
      fi
  fi
}

######################################

add_test_type hbaseanti

## @description  hbaseanti file filter
## @audience     private
## @stability    evolving
## @param        filename
function hbaseanti_filefilter
{
  local filename=$1

  if [[ ${filename} =~ \.java$ ]]; then
    add_test hbaseanti
  fi
}

## @description  hbaseanti patch file check
## @audience     private
## @stability    evolving
## @param        filename
function hbaseanti_patchfile
{
  local patchfile=$1
  local warnings
  local result

  if [[ "${BUILDMODE}" = full ]]; then
    return 0
  fi

  if ! verify_needed_test hbaseanti; then
    return 0
  fi

  big_console_header "Checking for known anti-patterns"

  start_clock

  warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
  if [[ ${warnings} -gt 0 ]]; then
    add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted."
    ((result=result+1))
  fi

  if [[ ${result} -gt 0 ]]; then
    return 1
  fi

  add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
  return 0
}

## @description  process the javac output for generating WARNING/ERROR
## @audience     private
## @stability    evolving
## @param        input filename
## @param        output filename
# Override the default javac_logfilter so that we can do a sort before outputing the WARNING/ERROR.
# This is because that the output order of the error prone warnings is not stable, so the diff
# method will report unexpected errors if we do not sort it. Notice that a simple sort will cause
# line number being sorted by lexicographical so the output maybe a bit strange to human but it is
# really hard to sort by file name first and then line number and column number in shell...
function hbase_javac_logfilter
{
  declare input=$1
  declare output=$2

  ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
}