| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| pipeline { |
| |
| agent { |
| label 'Hadoop' |
| } |
| |
| options { |
| // N.B. this is per-branch, which means per PR |
| disableConcurrentBuilds() |
| buildDiscarder(logRotator(numToKeepStr: '15')) |
| timeout (time: 7, unit: 'HOURS') |
| timestamps() |
| checkoutToSubdirectory('src') |
| } |
| |
| environment { |
| SOURCEDIR = 'src' |
| // will also need to change notification section below |
| PATCHDIR = 'out' |
| DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" |
| YETUS='yetus' |
| // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' |
| YETUS_VERSION='rel/0.11.1' |
| } |
| |
| parameters { |
| booleanParam(name: 'DEBUG', |
| defaultValue: false, |
| description: 'Print extra outputs for debugging the jenkins job and yetus') |
| } |
| |
| stages { |
| stage ('install yetus') { |
| steps { |
| dir("${WORKSPACE}/${YETUS}") { |
| checkout([ |
| $class: 'GitSCM', |
| branches: [[name: "${env.YETUS_VERSION}"]], |
| userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]] |
| ) |
| } |
| } |
| } |
| |
| stage ('precommit-run') { |
| steps { |
| withCredentials( |
| [usernamePassword(credentialsId: 'apache-hbase-at-github.com', |
| passwordVariable: 'GITHUB_PASSWORD', |
| usernameVariable: 'GITHUB_USER'), |
| usernamePassword(credentialsId: 'hbaseqa-at-asf-jira', |
| passwordVariable: 'JIRA_PASSWORD', |
| usernameVariable: 'JIRA_USER')]) { |
| sh '''#!/usr/bin/env bash |
| set -e |
| TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" |
| # this must be clean for every run |
| if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then |
| rm -rf "${WORKSPACE}/${PATCHDIR}" |
| fi |
| mkdir -p "${WORKSPACE}/${PATCHDIR}" |
| |
| ## Checking on H* machine nonsense |
| echo "JAVA_HOME: ${JAVA_HOME}" |
| ls -l "${JAVA_HOME}" || true |
| echo "MAVEN_HOME: ${MAVEN_HOME}" |
| echo "maven version:" |
| mvn --offline --version || true |
| echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/" |
| mkdir "${PATCHDIR}/machine" |
| cat /proc/cpuinfo >"${PATCHDIR}/machine/cpuinfo" 2>&1 || true |
| cat /proc/meminfo >"${PATCHDIR}/machine/meminfo" 2>&1 || true |
| cat /proc/diskstats >"${PATCHDIR}/machine/diskstats" 2>&1 || true |
| cat /sys/block/sda/stat >"${PATCHDIR}/machine/sys-block-sda-stat" 2>&1 || true |
| df -h >"${PATCHDIR}/machine/df-h" 2>&1 || true |
| ps -Awwf >"${PATCHDIR}/machine/ps-Awwf" 2>&1 || true |
| ifconfig -a >"${PATCHDIR}/machine/ifconfig-a" 2>&1 || true |
| lsblk -ta >"${PATCHDIR}/machine/lsblk-ta" 2>&1 || true |
| lsblk -fa >"${PATCHDIR}/machine/lsblk-fa" 2>&1 || true |
| cat /proc/loadavg >"${output}/loadavg" 2>&1 || true |
| ulimit -a >"${PATCHDIR}/machine/ulimit-a" 2>&1 || true |
| ## /H* |
| |
| # If CHANGE_URL is set (e.g., Github Branch Source plugin), process it. |
| # Otherwise exit, because we don't want HBase to do a |
| # full build. We wouldn't normally do this check for smaller |
| # projects. :) |
| if [[ -z "${CHANGE_URL}" ]]; then |
| echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html" |
| exit 0 |
| fi |
| # enable debug output for yetus |
| if [[ "true" = "${DEBUG}" ]]; then |
| YETUS_ARGS+=("--debug") |
| fi |
| # If we're doing docker, make sure we don't accidentally pollute the image with a host java path |
| if [ -n "${JAVA_HOME}" ]; then |
| unset JAVA_HOME |
| fi |
| YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") |
| # where the source is located |
| YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") |
| # our project defaults come from a personality file |
| # which will get loaded automatically by setting the project name |
| YETUS_ARGS+=("--project=hbase") |
| # lots of different output formats |
| YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") |
| YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") |
| YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") |
| # enable writing back to Github |
| YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}") |
| YETUS_ARGS+=(--github-user=${GITHUB_USER}) |
| # enable writing back to ASF JIRA |
| YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}") |
| YETUS_ARGS+=(--jira-user="${JIRA_USER}") |
| # auto-kill any surefire stragglers during unit test runs |
| YETUS_ARGS+=("--reapermode=kill") |
| YETUS_ARGS+=("--multijdktests=compile") |
| # set relatively high limits for ASF machines |
| # changing these to higher values may cause problems |
| # with other jobs on systemd-enabled machines |
| YETUS_ARGS+=("--proclimit=10000") |
| YETUS_ARGS+=("--dockermemlimit=20g") |
| # -1 findbugs issues that show up prior to the patch being applied |
| YETUS_ARGS+=("--findbugs-strict-precheck") |
| # rsync these files back into the archive dir |
| YETUS_ARGS+=("--archive-list=rat.txt") |
| # URL for user-side presentation in reports and such to our artifacts |
| # (needs to match the archive bits below) |
| YETUS_ARGS+=("--build-url-artifacts=artifact/out") |
| # plugins to enable |
| YETUS_ARGS+=("--plugins=all") |
| # don't let these tests cause -1s because we aren't really paying that |
| # much attention to them |
| YETUS_ARGS+=("--tests-filter=ruby-lint,test4tests") |
| # run in docker mode and specifically point to our |
| # Dockerfile since we don't want to use the auto-pulled version. |
| YETUS_ARGS+=("--docker") |
| YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") |
| YETUS_ARGS+=("--mvn-custom-repos") |
| YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64") |
| YETUS_ARGS+=("--findbugs-home=/usr") |
| YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*") |
| YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*") |
| YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh") |
| YETUS_ARGS+=("--quick-hadoopcheck") |
| YETUS_ARGS+=("--skip-errorprone") |
| # effectively treat dev-support as a custom maven module |
| YETUS_ARGS+=("--skip-dirs=dev-support") |
| # help keep the ASF boxes clean |
| YETUS_ARGS+=("--sentinel") |
| # use emoji vote so it is easier to find the broken line |
| YETUS_ARGS+=("--github-use-emoji-vote") |
| "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" |
| ''' |
| } |
| } |
| } |
| |
| } |
| |
| post { |
| always { |
| script { |
| // Yetus output |
| archiveArtifacts "${env.PATCHDIR}/**" |
| // Publish the HTML report so that it can be looked at |
| // Has to be relative to WORKSPACE. |
| publishHTML (target: [ |
| allowMissing: true, |
| keepAll: true, |
| alwaysLinkToLastBuild: true, |
| // Has to be relative to WORKSPACE |
| reportDir: "${env.PATCHDIR}", |
| reportFiles: 'report.html', |
| reportName: 'Yetus Report' |
| ]) |
| // Publish JUnit results |
| try { |
| junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml" |
| } catch(e) { |
| echo 'junit processing: ' + e.toString() |
| } |
| } |
| } |
| |
| // Jenkins pipeline jobs fill slaves on PRs without this :( |
| cleanup() { |
| script { |
| sh ''' |
| # See YETUS-764 |
| if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then |
| echo "test-patch process appears to still be running: killing" |
| kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true |
| sleep 10 |
| fi |
| if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then |
| echo "test-patch container appears to still be running: killing" |
| docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true |
| fi |
| # See HADOOP-13951 |
| chmod -R u+rxw "${WORKSPACE}" |
| ''' |
| deleteDir() |
| } |
| } |
| } |
| } |