| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| def getGithubAndJiraCreds() { |
| return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', |
| passwordVariable: 'GITHUB_TOKEN', |
| usernameVariable: 'GITHUB_USER'), |
| usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', |
| passwordVariable: 'JIRA_PASSWORD', |
| usernameVariable: 'JIRA_USER')] |
| } |
| |
| // Publish JUnit results only if there are XML files under surefire-reports |
| def publishJUnitResults() { |
| def findCmdExitCode = sh script: "find ${SOURCEDIR} -wholename */target/surefire-reports/*.xml | egrep .", returnStatus: true |
| boolean surefireReportsExist = findCmdExitCode == 0 |
| if (surefireReportsExist) { |
| echo "XML files found under surefire-reports, running junit" |
| // The path should be relative to WORKSPACE for the junit. |
| SRC = "${SOURCEDIR}/**/target/surefire-reports/*.xml".replace("$WORKSPACE/","") |
| try { |
| junit "${SRC}" |
| } catch(e) { |
| echo 'junit processing: ' + e.toString() |
| } |
| } else { |
| echo "No XML files found under surefire-reports, skipping junit" |
| } |
| } |
| |
| pipeline { |
| |
| agent { |
| label 'Hadoop' |
| } |
| |
| options { |
| buildDiscarder(logRotator(numToKeepStr: '5')) |
| timeout (time: 24, unit: 'HOURS') |
| timestamps() |
| checkoutToSubdirectory('src') |
| } |
| |
| environment { |
| YETUS='yetus' |
| // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' |
| YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f' |
| } |
| |
| parameters { |
| string(name: 'JIRA_ISSUE_KEY', |
| defaultValue: '', |
| description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234') |
| } |
| |
| stages { |
| stage ('install yetus') { |
| steps { |
| dir("${WORKSPACE}/${YETUS}") { |
| checkout([ |
| $class: 'GitSCM', |
| branches: [[name: "${env.YETUS_VERSION}"]], |
| userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]] |
| ) |
| } |
| } |
| } |
| |
| // Setup codebase so that each platform's build happens in its own exclusive copy of the |
| // codebase. |
| // Primarily because YETUS messes up the git branch information and affects the subsequent |
| // optional stages after the first one. |
| stage ('setup sources') { |
| steps { |
| dir("${WORKSPACE}/centos-7") { |
| sh '''#!/usr/bin/env bash |
| |
| cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7 |
| ''' |
| } |
| |
| dir("${WORKSPACE}/centos-8") { |
| sh '''#!/usr/bin/env bash |
| |
| cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8 |
| ''' |
| } |
| |
| dir("${WORKSPACE}/debian-10") { |
| sh '''#!/usr/bin/env bash |
| |
| cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10 |
| ''' |
| } |
| |
| dir("${WORKSPACE}/ubuntu-focal") { |
| sh '''#!/usr/bin/env bash |
| |
| cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal |
| ''' |
| } |
| } |
| } |
| |
| // This is an optional stage which runs only when there's a change in |
| // C++/C++ build/platform. |
| // This stage serves as a means of cross platform validation, which is |
| // really needed to ensure that any C++ related/platform change doesn't |
| // break the Hadoop build on Centos 7. |
| stage ('precommit-run Centos 7') { |
| environment { |
| SOURCEDIR = "${WORKSPACE}/centos-7/src" |
| PATCHDIR = "${WORKSPACE}/centos-7/out" |
| DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7" |
| IS_OPTIONAL = 1 |
| } |
| |
| steps { |
| withCredentials(getGithubAndJiraCreds()) { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" run_ci |
| ''' |
| } |
| } |
| |
| post { |
| // Since this is an optional platform, we want to copy the artifacts |
| // and archive it only if the build fails, to help with debugging. |
| failure { |
| sh '''#!/usr/bin/env bash |
| |
| cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}" |
| ''' |
| archiveArtifacts "out/**" |
| } |
| |
| cleanup() { |
| script { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc |
| ''' |
| } |
| } |
| } |
| } |
| |
| // This is an optional stage which runs only when there's a change in |
| // C++/C++ build/platform. |
| // This stage serves as a means of cross platform validation, which is |
| // really needed to ensure that any C++ related/platform change doesn't |
| // break the Hadoop build on Centos 8. |
| stage ('precommit-run Centos 8') { |
| environment { |
| SOURCEDIR = "${WORKSPACE}/centos-8/src" |
| PATCHDIR = "${WORKSPACE}/centos-8/out" |
| DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8" |
| IS_OPTIONAL = 1 |
| } |
| |
| steps { |
| withCredentials(getGithubAndJiraCreds()) { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" run_ci |
| ''' |
| } |
| } |
| |
| post { |
| // Since this is an optional platform, we want to copy the artifacts |
| // and archive it only if the build fails, to help with debugging. |
| failure { |
| sh '''#!/usr/bin/env bash |
| |
| cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}" |
| ''' |
| archiveArtifacts "out/**" |
| } |
| |
| cleanup() { |
| script { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc |
| ''' |
| } |
| } |
| } |
| } |
| |
| // This is an optional stage which runs only when there's a change in |
| // C++/C++ build/platform. |
| // This stage serves as a means of cross platform validation, which is |
| // really needed to ensure that any C++ related/platform change doesn't |
| // break the Hadoop build on Debian 10. |
| stage ('precommit-run Debian 10') { |
| environment { |
| SOURCEDIR = "${WORKSPACE}/debian-10/src" |
| PATCHDIR = "${WORKSPACE}/debian-10/out" |
| DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" |
| IS_OPTIONAL = 1 |
| } |
| |
| steps { |
| withCredentials(getGithubAndJiraCreds()) { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" run_ci |
| ''' |
| } |
| } |
| |
| post { |
| // Since this is an optional platform, we want to copy the artifacts |
| // and archive it only if the build fails, to help with debugging. |
| failure { |
| sh '''#!/usr/bin/env bash |
| |
| cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" |
| ''' |
| archiveArtifacts "out/**" |
| } |
| |
| cleanup() { |
| script { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc |
| ''' |
| } |
| } |
| } |
| } |
| |
| // We want to use Ubuntu Focal as our main CI and thus, this stage |
| // isn't optional (runs for all the PRs). |
| stage ('precommit-run Ubuntu focal') { |
| environment { |
| SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src" |
| PATCHDIR = "${WORKSPACE}/ubuntu-focal/out" |
| DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" |
| IS_OPTIONAL = 0 |
| } |
| |
| steps { |
| withCredentials(getGithubAndJiraCreds()) { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" run_ci |
| ''' |
| } |
| } |
| |
| post { |
| always { |
| script { |
| // Publish status if it was missed (YETUS-1059) |
| withCredentials( |
| [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd', |
| passwordVariable: 'GITHUB_TOKEN', |
| usernameVariable: 'GITHUB_USER')]) { |
| sh '''#!/usr/bin/env bash |
| |
| # Copy the artifacts of Ubuntu focal build to workspace |
| cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}" |
| |
| # Send Github status |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery |
| ''' |
| } |
| |
| // YETUS output |
| archiveArtifacts "out/**" |
| |
| // Publish the HTML report so that it can be looked at |
| // Has to be relative to WORKSPACE. |
| publishHTML (target: [ |
| allowMissing: true, |
| keepAll: true, |
| alwaysLinkToLastBuild: true, |
| // Has to be relative to WORKSPACE |
| reportDir: "out", |
| reportFiles: 'report.html', |
| reportName: 'Yetus Report' |
| ]) |
| |
| publishJUnitResults() |
| } |
| } |
| |
| cleanup() { |
| script { |
| sh '''#!/usr/bin/env bash |
| |
| chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" |
| "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc |
| ''' |
| } |
| } |
| } |
| } |
| } |
| |
| post { |
| // Jenkins pipeline jobs fill slaves on PRs without this :( |
| cleanup() { |
| script { |
| sh '''#!/usr/bin/env bash |
| |
| # See HADOOP-13951 |
| chmod -R u+rxw "${WORKSPACE}" |
| ''' |
| deleteDir() |
| } |
| } |
| } |
| } |
