| #!/usr/bin/env bash |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # |
| # Based on jenkins_precommit_jira_yetus.sh of the HBase project |
| # |
| |
| if [[ "true" = "${DEBUG}" ]]; then |
| set -x |
| printenv |
| fi |
| |
| ##To set jenkins Environment Variables: |
| export TOOLS_HOME=/home/jenkins/tools |
| #export JAVA_HOME=${JAVA_HOME_HADOOP_MACHINES_HOME} |
| export CLOVER_HOME=${TOOLS_HOME}/clover/latest |
| #export MAVEN_HOME=${MAVEN_3_0_4_HOME} |
| export MAVEN_HOME=/home/jenkins/tools/maven/apache-maven-3.0.5 |
| |
| #export PATH=$PATH:${JAVA_HOME}/bin:${MAVEN_HOME}/bin: |
| export PATH=$PATH:${MAVEN_HOME}/bin: |
| |
| export MAVEN_OPTS=-Xmx3g |
| |
| YETUS_RELEASE=0.12.0 |
| COMPONENT=${WORKSPACE}/component |
| TEST_FRAMEWORK=${WORKSPACE}/test_framework |
| |
| PATCHPROCESS=${WORKSPACE}/patchprocess |
| if [[ -d ${PATCHPROCESS} ]]; then |
| echo "[WARN] patch process already existed '${PATCHPROCESS}'" |
| rm -rf "${PATCHPROCESS}" |
| fi |
| mkdir -p "${PATCHPROCESS}" |
| |
| |
| ## Checking on H* machine nonsense |
| echo "JAVA_HOME: ${JAVA_HOME}" |
| ls -l "${JAVA_HOME}" || true |
| echo "MAVEN_HOME: ${MAVEN_HOME}" |
| echo "maven version:" |
| mvn --offline --version || true |
| echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/" |
| mkdir "${PATCHPROCESS}/machine" |
| cat /proc/cpuinfo >"${PATCHPROCESS}/machine/cpuinfo" 2>&1 || true |
| cat /proc/meminfo >"${PATCHPROCESS}/machine/meminfo" 2>&1 || true |
| cat /proc/diskstats >"${PATCHPROCESS}/machine/diskstats" 2>&1 || true |
| cat /sys/block/sda/stat >"${PATCHPROCESS}/machine/sys-block-sda-stat" 2>&1 || true |
| df -h >"${PATCHPROCESS}/machine/df-h" 2>&1 || true |
| ps -Awwf >"${PATCHPROCESS}/machine/ps-Awwf" 2>&1 || true |
| ifconfig -a >"${PATCHPROCESS}/machine/ifconfig-a" 2>&1 || true |
| lsblk -ta >"${PATCHPROCESS}/machine/lsblk-ta" 2>&1 || true |
| lsblk -fa >"${PATCHPROCESS}/machine/lsblk-fa" 2>&1 || true |
| cat /proc/loadavg >"${PATCHPROCESS}/loadavg" 2>&1 || true |
| ulimit -a >"${PATCHPROCESS}/machine/ulimit-a" 2>&1 || true |
| |
| ## /H* |
| |
| ### Download Yetus |
| if [[ "true" != "${USE_YETUS_PRERELEASE}" ]]; then |
| if [ ! -d "${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}" ]; then |
| mkdir -p "${TEST_FRAMEWORK}" |
| cd "${TEST_FRAMEWORK}" || exit 1 |
| # clear out any cached 'use a prerelease' versions |
| rm -rf apache-yetus-* |
| |
| mkdir -p "${TEST_FRAMEWORK}/.gpg" |
| chmod -R 700 "${TEST_FRAMEWORK}/.gpg" |
| |
| curl -L --fail -o "${TEST_FRAMEWORK}/KEYS_YETUS" https://dist.apache.org/repos/dist/release/yetus/KEYS |
| gpg --homedir "${TEST_FRAMEWORK}/.gpg" --import "${TEST_FRAMEWORK}/KEYS_YETUS" |
| |
| ## Release |
| curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz" |
| curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc" |
| gpg --homedir "${TEST_FRAMEWORK}/.gpg" --verify "apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc" |
| tar xzpf "apache-yetus-${YETUS_RELEASE}-bin.tar.gz" |
| fi |
| TESTPATCHBIN=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/bin/test-patch |
| TESTPATCHLIB=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/lib/precommit |
| else |
| prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*") |
| if [ ! -d "${prerelease_dirs[0]}" ]; then |
| mkdir -p "${TEST_FRAMEWORK}" |
| cd "${TEST_FRAMEWORK}" || exit |
| ## from github |
| curl -L --fail "https://api.github.com/repos/${YETUS_PRERELEASE_GITHUB}/tarball/HEAD" > yetus.tar.gz |
| tar xvpf yetus.tar.gz |
| prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*") |
| fi |
| TESTPATCHBIN=${prerelease_dirs[0]}/precommit/test-patch.sh |
| TESTPATCHLIB=${prerelease_dirs[0]}/precommit |
| fi |
| |
| if [[ "true" = "${DEBUG}" ]]; then |
| # DEBUG print the test framework |
| ls -l "${TESTPATCHBIN}" |
| ls -la "${TESTPATCHLIB}/test-patch.d/" |
| # DEBUG print the local customization |
| if [ -d "${COMPONENT}/dev/test-patch.d" ]; then |
| ls -la "${COMPONENT}/dev/test-patch.d/" |
| fi |
| YETUS_ARGS=(--debug "${YETUS_ARGS[@]}") |
| fi |
| |
| if [ -r "${COMPONENT}/dev/phoenix-personality.sh" ] ; then |
| YETUS_ARGS=("--personality=${COMPONENT}/dev/phoenix-personality.sh" "${YETUS_ARGS[@]}") |
| fi |
| |
| if [ ! -x "${TESTPATCHBIN}" ] && [ -n "${TEST_FRAMEWORK}" ] && [ -d "${TEST_FRAMEWORK}" ]; then |
| echo "Something is amiss with the test framework; removing it. please re-run." |
| rm -rf "${TEST_FRAMEWORK}" |
| exit 1 |
| fi |
| |
| cd "${WORKSPACE}" || exit |
| |
| |
| # |
| # Yetus *always* builds with JAVA_HOME, so no need to list it. |
| # |
| # non-docker-mode JDK: |
| |
| # docker-mode: (openjdk 7 added for free) |
| # --docker \ |
| # --multijdkdirs="/usr/lib/jvm/java-8-openjdk-amd64" \ |
| |
| if [[ "true" = "${RUN_IN_DOCKER}" ]]; then |
| YETUS_ARGS=( |
| --docker \ |
| --dockermemlimit=20g \ |
| "--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64" \ |
| "${YETUS_ARGS[@]}" \ |
| ) |
| if [ -r "${COMPONENT}/dev/docker/Dockerfile.yetus" ]; then |
| YETUS_ARGS=("--dockerfile=${COMPONENT}/dev/docker/Dockerfile.yetus" "${YETUS_ARGS[@]}") |
| fi |
| fi |
| |
| if [ -d "${COMPONENT}/dev/test-patch.d" ]; then |
| YETUS_ARGS=("--user-plugins=${COMPONENT}/dev/test-patch.d" "${YETUS_ARGS[@]}") |
| fi |
| |
| #Removed some code not relevant to Phoenix |
| |
| YETUS_ARGS=("--skip-dirs=dev" "${YETUS_ARGS[@]}") |
| |
| /bin/bash "${TESTPATCHBIN}" \ |
| "${YETUS_ARGS[@]}" \ |
| --patch-dir="${PATCHPROCESS}" \ |
| --basedir="${COMPONENT}" \ |
| --mvn-custom-repos \ |
| --mvn-custom-repos-dir="${WORKSPACE}/yetus-m2/" \ |
| --whitespace-eol-ignore-list=".*/generated/.*" \ |
| --whitespace-tabs-ignore-list=".*/generated/.*" \ |
| --run-tests \ |
| --plugins="all,-findbugs,-gitlab" \ |
| --html-report-file=report.html \ |
| --jira-user=hadoopqa \ |
| --jira-password="${JIRA_PASSWORD}" \ |
| "PHOENIX-${ISSUE_NUM}" |
| |
| find "${COMPONENT}" -name target -exec chmod -R u+w {} \; |