HBASE-24642: Apache Yetus test-patch and Jenkins integration. (#10)
Adds the plumbing needed to run test-patch tooling from Yetus.
Sample output (with a few failing tests).
| Vote | Subsystem | Runtime | Comment
============================================================================
+---------------------------------------------------------------------------
| | | | Prechecks
+---------------------------------------------------------------------------
+---------------------------------------------------------------------------
| | | | master Compile Tests
+---------------------------------------------------------------------------
| +1 | CMakeLists.txt | 2m 9s | master passed
| +1 | compile | 0m 44s | master passed
+---------------------------------------------------------------------------
| | | | Patch Compile Tests
+---------------------------------------------------------------------------
| +1 | CMakeLists.txt | 2m 3s | the patch passed
| +1 | compile | 13m 54s | the patch passed
+---------------------------------------------------------------------------
| | | | Other Tests
+---------------------------------------------------------------------------
| -1 | unit | 0m 34s | root in the patch failed.
| | | 21m 1s |
Reason | Tests
Failed CTEST tests | async-batch-rpc-retrying-test
| async-rpc-retrying-test
| client-test
| filter-test
| location-cache-retry-test
| location-cache-test
| scanner-test
Supports both docker and non-docker mode. Runs all the unit-tests
as a part of the test-patch run.
Additionally adds the Jenkins integration needed to run the precommit
on Apache infra. To be hooked up with the following job
https://builds.apache.org/view/H-L/view/HBase/job/hbase-native-client-github-precommit/
Signed-off-by: Marc Parisi <phrocker@apache.org>
Signed-off-by: Josh Elser <elserj@apache.org>
diff --git a/.gitignore b/.gitignore
index 794447c..93c6522 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,11 @@
*.swp
+# build directories
+build*/*
+cmake-build-debug/*
+dependencies/*
+
# Thirdparty dirs
third-party/*
/gcc-debug/
@@ -25,6 +30,7 @@
# Tests
*-test
+src/test/target/*
# Executables
load-client
diff --git a/bin/hbase-native-client-personality.sh b/bin/hbase-native-client-personality.sh
new file mode 100755
index 0000000..5324703
--- /dev/null
+++ b/bin/hbase-native-client-personality.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# You'll need a local installation of
+# [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
+# to use this personality.
+#
+# Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
+# build from it, or use the convenience binaries provided on that download page.
+#
+# To run against, e.g. HBASE-15074 you'd then do
+# ```bash
+# test-patch --personality=bin/hbase-native-client-personality.sh HBASE-15074
+# ```
+#
+# If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
+# ```bash
+# test-patch --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
+# ````
+#
+# pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
+# directory / branch in order to have things match what the issue patch requests.
+
+# Plugins enabled.
+personality_plugins "compile,unit,cmake,asflicense,github,jira,ctest,htmlout"
+
+if ! declare -f "yetus_info" >/dev/null; then
+ function yetus_info
+ {
+ echo "[$(date) INFO]: $*" 1>&2
+ }
+fi
+
+# work around yetus overwriting JAVA_HOME from our docker image
+function docker_do_env_adds
+{
+ declare k
+ for k in "${DOCKER_EXTRAENVS[@]}"; do
+ if [[ "JAVA_HOME" == "${k}" ]]; then
+ if [ -n "${JAVA_HOME}" ]; then
+ DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
+ fi
+ else
+ DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
+ fi
+ done
+ DOCKER_EXTRAARGS+=("-h=securecluster")
+}
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ SUDO_USER=root
+ # shellcheck disable=SC2034
+ BUILD_NATIVE=true
+ # shellcheck disable=SC2034
+ BUILDTOOL=cmake
+ # shellcheck disable=SC2034
+ # Passed to cmake command using a custom personality.
+ CMAKE_ARGS="-DDOWNLOAD_DEPENDENCIES=ON"
+ # shellcheck disable=SC2034
+ # Expected by Yetus for compiling non-jvm projects.
+ JVM_REQUIRED=false
+ #shellcheck disable=SC2034
+ PROJECT_NAME=hbase-native-client
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^HBASE-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/hbase-native-client"
+ # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
+ # Up it. See HBASE-19902 for how we arrived at this number.
+ #shellcheck disable=SC2034
+ PROCLIMIT=10000
+ # Override if you want to bump up the memlimit for docker.
+ # shellcheck disable=SC2034
+ DOCKERMEMLIMIT=4g
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local args
+ yetus_info "Personality: ${repostatus} ${testtype}"
+ clear_personality_queue
+ if [[ "${testtype}" =~ CMakeLists.txt ]]; then
+ args=${CMAKE_ARGS}
+ yetus_debug "Appending CMake args ${args}"
+ fi
+ personality_enqueue_module . ${args}
+}
diff --git a/bin/jenkins/Jenkinsfile_GitHub b/bin/jenkins/Jenkinsfile_GitHub
new file mode 100644
index 0000000..60c4cb0
--- /dev/null
+++ b/bin/jenkins/Jenkinsfile_GitHub
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pipeline {
+
+ agent {
+ label 'Hadoop'
+ }
+
+ options {
+ // N.B. this is per-branch, which means per PR
+ disableConcurrentBuilds()
+ buildDiscarder(logRotator(numToKeepStr: '15'))
+ timeout (time: 7, unit: 'HOURS')
+ timestamps()
+ skipDefaultCheckout()
+ }
+
+ environment {
+ SRC_REL = 'src'
+ PATCH_REL = 'output'
+ YETUS_REL = 'yetus'
+ // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
+ YETUS_VERSION = 'rel/0.12.0'
+ DOCKERFILE_REL = "${SRC_REL}/docker-files/Dockerfile"
+ YETUS_DRIVER_REL = "${SRC_REL}/bin/run_tests_with_yetus.sh"
+ ARCHIVE_PATTERN_LIST = 'org.apache.h*.txt,*.dumpstream,*.dump,*.jstack,*.pstack,*.core,hs_err*.log'
+ BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
+ }
+
+ parameters {
+ booleanParam(name: 'DEBUG',
+ defaultValue: false,
+ description: 'Print extra outputs for debugging the jenkins job and yetus')
+ }
+
+ stages {
+ stage ('precommit checks') {
+ parallel {
+ stage ('yetus patch ctests') {
+ agent {
+ node {
+ label 'Hadoop'
+ }
+ }
+ environment {
+ WORKDIR_REL = "hbase-native-client-precommit"
+ WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
+ SOURCEDIR = "${WORKDIR}/${SRC_REL}"
+ PATCHDIR = "${WORKDIR}/${PATCH_REL}"
+ BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
+ DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
+ YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+ YETUSDIR = "${WORKDIR}/${YETUS_REL}"
+ }
+ steps {
+ dir("${SOURCEDIR}") {
+ checkout scm
+ }
+ dir("${YETUSDIR}") {
+ checkout([
+ $class : 'GitSCM',
+ branches : [[name: "${YETUS_VERSION}"]],
+ userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
+ )
+ }
+ dir("${WORKDIR}") {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'apache-hbase-at-github.com',
+ passwordVariable: 'GITHUB_PASSWORD',
+ usernameVariable: 'GITHUB_USER'
+ )]) {
+ sh label: 'test-patch', script: '''
+ hostname -a ; pwd ; ls -la
+ printenv 2>&1 | sort
+ echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
+ "${YETUS_DRIVER}"
+ '''
+ }
+ }
+ }
+ post {
+ always {
+ sh label: 'zip surefire reports', script: '''
+ if [ -d "${PATCHDIR}/archiver" ]; then
+ count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
+ '''
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "${WORKDIR_REL}/${PATCH_REL}",
+ reportFiles: 'report.html',
+ reportName: 'PR ctests report'
+ ]
+ }
+ // Jenkins pipeline jobs fill slaves on PRs without this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script: '''
+ # See YETUS-764
+ if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+ echo "test-patch process appears to still be running: killing"
+ kill `cat "${PATCHDIR}/pidfile.txt"` || true
+ sleep 10
+ fi
+ if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+ echo "test-patch container appears to still be running: killing"
+ docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
+ fi
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ dir ("${WORKDIR}") {
+ deleteDir()
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ post {
+ // Jenkins pipeline jobs fill hosts on PRs without this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script: '''
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ deleteDir()
+ }
+ }
+ }
+}
diff --git a/bin/jenkins/gather_machine_environment.sh b/bin/jenkins/gather_machine_environment.sh
new file mode 100755
index 0000000..fa99d3f
--- /dev/null
+++ b/bin/jenkins/gather_machine_environment.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} /path/for/output/dir"
+ echo ""
+ echo " Gather info about a build machine that test harnesses should poll before running."
+ echo " presumes you'll then archive the passed output dir."
+
+ exit 1
+}
+
+if [ "$#" -lt 1 ]; then
+ usage
+fi
+
+
+declare output=$1
+
+if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then
+ echo "Specified output directory must exist and be writable." >&2
+ exit 1
+fi
+
+echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
+echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
+ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
+echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
+mvn --offline --version >"${output}/mvn_version" 2>&1 || true
+cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
+cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
+df -h >"${output}/df-h" 2>&1 || true
+ps -Aww >"${output}/ps-Aww" 2>&1 || true
+ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
+lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
+lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
+ulimit -l >"${output}/ulimit-l" 2>&1 || true
+uptime >"${output}/uptime" 2>&1 || true
diff --git a/bin/run_tests_with_yetus.sh b/bin/run_tests_with_yetus.sh
new file mode 100755
index 0000000..633a234
--- /dev/null
+++ b/bin/run_tests_with_yetus.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eo pipefail
+
+if [[ "true" = "${DEBUG}" ]]; then
+ set -x
+ printenv
+fi
+
+declare -i missing_env=0
+declare -a required_envs=(
+ # these ENV variables define the required API with Jenkinsfile_GitHub
+ "ARCHIVE_PATTERN_LIST"
+ "BUILD_URL_ARTIFACTS"
+ "DOCKERFILE"
+ "GITHUB_PASSWORD"
+ "GITHUB_USER"
+ "PATCHDIR"
+ "SOURCEDIR"
+ "YETUSDIR"
+)
+# Validate params
+for required_env in "${required_envs[@]}"; do
+ if [ -z "${!required_env}" ]; then
+ echo "[ERROR] Required environment variable '${required_env}' is not set."
+ missing_env=${missing_env}+1
+ fi
+done
+
+if [ ${missing_env} -gt 0 ]; then
+ echo "[ERROR] Please set the required environment variables before invoking. If this error is " \
+ "on Jenkins, then please file a JIRA about the error."
+ exit 1
+fi
+
+# this must be clean for every run
+rm -rf "${PATCHDIR}"
+mkdir -p "${PATCHDIR}"
+
+# Gather machine information
+mkdir "${PATCHDIR}/machine"
+"${SOURCEDIR}/bin/jenkins/gather_machine_environment.sh" "${PATCHDIR}/machine"
+
+YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}")
+YETUS_ARGS+=("--ignore-unknown-options=true")
+YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
+# where the source is located
+YETUS_ARGS+=("--basedir=${SOURCEDIR}")
+# lots of different output formats
+YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
+YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
+# enable writing back to Github
+YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
+YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+# rsync these files back into the archive dir
+YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}")
+# URL for user-side presentation in reports and such to our artifacts
+YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}")
+# run in docker mode and specifically point to our
+YETUS_ARGS+=("--docker")
+YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+
+# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
+TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+
+if [[ "true" = "${DEBUG}" ]]; then
+ YETUS_ARGS=(--debug "${YETUS_ARGS[@]}")
+fi
+
+# Sanity checks for downloaded Yetus binaries.
+if [ ! -x "${TESTPATCHBIN}" ]; then
+ echo "Something is amiss with Yetus download."
+ exit 1
+fi
+
+# Runs in docker/non-docker mode. In docker mode, we pass the appropriate docker file with
+# all the dependencies installed as a part of init.
+if [[ "true" = "${RUN_IN_DOCKER}" ]]; then
+ YETUS_ARGS=(
+ --docker \
+ "--dockerfile=${SOURCEDIR}/docker-files/Dockerfile" \
+ "${YETUS_ARGS[@]}"
+ )
+fi
+
+echo "Using YETUS_ARGS: ${YETUS_ARGS[*]}"
+
+# shellcheck disable=SC2068
+/bin/bash "${TESTPATCHBIN}" \
+ --personality="${SOURCEDIR}/bin/hbase-native-client-personality.sh" \
+ --run-tests \
+ ${YETUS_ARGS[@]}
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 20db789..7c3c808 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -39,7 +39,7 @@
# Build the image
#
# This shouldn't be needed after the development environment is a little more stable.
-docker build -t hbase_native -f docker-files/Dockerfile .
+docker build -t hbase_native -f docker-files/Dockerfile docker-files
# After the image is built run the thing
docker run --privileged=true -h="securecluster" -p 16050:16050/tcp \
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index 0a735b4..3cad6d2 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -15,6 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# This Dockerfile is shared for both personal use as well as precommit testing. Certain changes might break Yetus
+# precommit test-patch integration, so be mindful of that when making changes.
+
FROM ubuntu:16.04
ARG CC=/usr/bin/gcc-5
@@ -71,10 +74,17 @@
echo 'xst -k hbase-host.keytab hbase/securecluster@EXAMPLE.COM' >> /tmp/krb-princ.pass ; \
kadmin.local < /tmp/krb-princ.pass ;
-COPY docker-files/krb5.conf /etc
+COPY krb5.conf /etc
RUN echo "enabled=1" >> /etc/default/apport
-ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && echo "/tmp/core.%h.%e.%t" >> /proc/sys/kernel/core_pattern && sysctl -p && ulimit -c unlimited && /bin/bash
-
WORKDIR /usr/src/hbase/hbase-native-client
+
+CMD ["/usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && sysctl -p && ulimit -c unlimited && /bin/bash"]
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
