// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

pipeline {

    agent {
        label 'hbase'
    }

    options {
        // N.B. this is per-branch, which means per PR
        disableConcurrentBuilds()
        buildDiscarder(logRotator(numToKeepStr: '15'))
        timeout (time: 1, unit: 'HOURS')
        timestamps()
        skipDefaultCheckout()
    }

    parameters {
        booleanParam(name: 'DEBUG',
               defaultValue: false,
               description: 'Print extra outputs for debugging the jenkins job and yetus')
        // the hbase and hadoop versions listed here need to match the matrix axes in the test
        // section. it's not currently possible to reuse a single array for both purposes.
        choice(name: 'HBASE_VERSION',
               choices: ['all', '2.2.6', '2.3.3'],
               description: 'HBase releases to test. default is everything in the list.')
        choice(name: 'HADOOP_VERSION',
               choices: ['all', '3.2.1', '2.10.0'],
               description: 'Hadoop versions to run each hbase version on. default is everything in the list.')
    }

    environment {
        SRC_REL = 'src'
        PATCH_REL = 'output'
        YETUS_REL = 'yetus'
        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
        YETUS_VERSION = 'rel/0.12.0'
        DOCKERFILE_REL = "${SRC_REL}/dev-support/jenkins/Dockerfile"
        YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins/jenkins_precommit_github_yetus.sh"
        ARCHIVE_PATTERN_LIST = '*.dump'
        SET_JAVA_HOME = '/usr/local/openjdk-8'
        PLUGINS = 'all'
    }


    stages {
        stage ('precommit checks') {
            when {
                changeRequest()
            }
            environment {
                WORKDIR_REL = 'yetus-precommit-check'
                WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
                SOURCEDIR = "${WORKDIR}/${SRC_REL}"
                PATCHDIR = "${WORKDIR}/${PATCH_REL}"
                DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
                YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
                YETUSDIR = "${WORKDIR}/${YETUS_REL}"
                BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
            }
            steps {
                dir("${SOURCEDIR}") {
                    checkout scm
                }
                dir("${YETUSDIR}") {
                    checkout([
                      $class           : 'GitSCM',
                      branches         : [[name: "${YETUS_VERSION}"]],
                      userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
                    )
                }
                dir("${WORKDIR}") {
                    withCredentials([
                      usernamePassword(
                        credentialsId: 'apache-hbase-at-github.com',
                        passwordVariable: 'GITHUB_PASSWORD',
                        usernameVariable: 'GITHUB_USER'
                      )]) {
                        sh label: 'test-patch', script: '''#!/bin/bash -e
                            printenv 2>&1 | sort
                            echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
                            "${YETUS_DRIVER}"
                        '''
                    }
                }
            }
            post {
                always {
                    // Has to be relative to WORKSPACE.
                    archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
                    archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
                    publishHTML target: [
                      allowMissing: true,
                      keepAll: true,
                      alwaysLinkToLastBuild: true,
                      // Has to be relative to WORKSPACE
                      reportDir: "${WORKDIR_REL}/${PATCH_REL}",
                      reportFiles: 'report.html',
                      reportName: 'PR General Check Report'
                    ]
                }
                // Jenkins pipeline jobs fill slaves on PRs without this :(
                cleanup() {
                    script {
                        sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
                            # See YETUS-764
                            if [ -f "${PATCHDIR}/pidfile.txt" ]; then
                              echo "test-patch process appears to still be running: killing"
                              kill `cat "${PATCHDIR}/pidfile.txt"` || true
                              sleep 10
                            fi
                            if [ -f "${PATCHDIR}/cidfile.txt" ]; then
                              echo "test-patch container appears to still be running: killing"
                              docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
                            fi
                            # See HADOOP-13951
                            chmod -R u+rxw "${WORKSPACE}"
                        '''
                        dir ("${WORKDIR}") {
                            deleteDir()
                        }
                    }
                }
            }
        }
        stage ('noop htrace drop in') {
            when {
                anyOf {
                    changeset "hbase-noop-htrace/**"
                    changeset "dev-support/jenkins/Jenkinsfile"
                }
            }
            tools {
                // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
                jdk "jdk_1.8_latest"
            }
            stages {
                stage ('setup') {
                    tools {
                        maven 'maven_latest'
                    }
                    environment {
                        WORKDIR = "${WORKSPACE}/htrace-noop"
                        CACHE_DIR = "${WORKSPACE}/cache"
                    }
                    steps {
                        dir ("htrace-noop") {
                            dir ("component") {
                                echo 'Build the htrace replacement artifact.'
                                checkout scm
                                sh 'mvn -DskipTests -pl hbase-noop-htrace clean package'
                            }
                            dir ("tools") {
                                echo 'Downloading enabling scripts from main hbase repo.'
                                sh '''#!/usr/bin/env bash
                                    set -oe
                                    declare script
                                    declare -a needed_files
                                    needed_files=( \
                                        hbase_nightly_pseudo-distributed-test.sh \
                                        jenkins-scripts/cache-apache-project-artifact.sh \
                                    )
                                    for script in "${needed_files[@]}"; do
                                        curl -L -O https://raw.githubusercontent.com/apache/hbase/HEAD/dev-support/"${script}"
                                        chmod +x "$(basename "${script}")"
                                    done
                                '''
                            }
                            stash name: 'scripts', includes: "tools/hbase_nightly_pseudo-distributed-test.sh"
                            dir ("hbase") {
                                script {
                                    def hbase_versions = [ params.HBASE_VERSION ]
                                    if (params.HBASE_VERSION == 'all') {
                                        // this set needs to match the matrix axes below
                                        hbase_versions = [ '2.2.6', '2.3.3' ]
                                    }
                                    hbase_versions.each {
                                        def hbase_version = it
                                        sh """#!/usr/bin/env bash
                                            set -e
                                            set -x
                                            mkdir -p "downloads/hbase-${hbase_version}"
                                            mkdir -p "${CACHE_DIR}"
                                            echo 'downloading hbase version ${hbase_version}'
                                            '${WORKDIR}/tools/cache-apache-project-artifact.sh' \
                                                    --working-dir '${WORKDIR}/hbase/downloads/hbase-${hbase_version}' \
                                                    --keys 'https://downloads.apache.org/hbase/KEYS' \
                                                    '${CACHE_DIR}/hbase-${hbase_version}-bin.tar.gz' \
                                                    'hbase/${hbase_version}/hbase-${hbase_version}-bin.tar.gz'
                                            mkdir 'hbase-${hbase_version}'
                                            declare noop_htrace
                                            noop_htrace="\$(ls -1 '${WORKDIR}/component/hbase-noop-htrace/target/'hbase-noop-htrace-*.jar | head -n 1)"
                                            if [ -z "\${noop_htrace}" ]; then
                                              echo "failed to find htrace noop replacement. did building step work?" >&2
                                              exit 1
                                            fi
                                            '${WORKDIR}/component/dev-support/jenkins/swap_htrace_jar.sh' '${WORKDIR}/hbase/hbase-${hbase_version}' '${CACHE_DIR}/hbase-${hbase_version}-bin.tar.gz' "\${noop_htrace}"
                                        """
                                        stash name: "hbase-${hbase_version}", includes: "hbase-${hbase_version}/**"
                                    }
                                }
                            }
                            dir ("hadoop") {
                                script {
                                    def hadoop_versions = [ params.HADOOP_VERSION ]
                                    if (params.HADOOP_VERSION == 'all') {
                                        // this set needs to match the matrix axes below
                                        hadoop_versions = [ '3.2.1', '2.10.0']
                                    }
                                    hadoop_versions.each {
                                        def hadoop_version = it
                                        sh """#!/usr/bin/env bash
                                            set -e
                                            set -x
                                            mkdir -p "downloads/hadoop-${hadoop_version}"
                                            mkdir -p "${CACHE_DIR}"
                                            echo "downloading hadoop version ${hadoop_version}"
                                            "${WORKDIR}/tools/cache-apache-project-artifact.sh" \
                                                    --working-dir "${WORKDIR}/hadoop/downloads/hadoop-${hadoop_version}" \
                                                    --keys 'https://downloads.apache.org/hadoop/common/KEYS' \
                                                    "${CACHE_DIR}/hadoop-${hadoop_version}-bin.tar.gz" \
                                                    "hadoop/common/hadoop-${hadoop_version}/hadoop-${hadoop_version}.tar.gz"
                                            mkdir "hadoop-${hadoop_version}"
                                            declare noop_htrace
                                            noop_htrace="\$(ls -1 "${WORKDIR}"/component/hbase-noop-htrace/target/hbase-noop-htrace-*.jar | head -n 1)"
                                            if [ -z "\${noop_htrace}" ]; then
                                              echo "failed to find htrace noop replacement. did building step work?" >&2
                                              exit 1
                                            fi
                                            '${WORKDIR}/component/dev-support/jenkins/swap_htrace_jar.sh' "${WORKDIR}/hadoop/hadoop-${hadoop_version}" "${CACHE_DIR}/hadoop-${hadoop_version}-bin.tar.gz" "\${noop_htrace}"
                                        """
                                        stash name: "hadoop-${hadoop_version}", includes: "hadoop-${hadoop_version}/**"
                                    }
                                }
                            }
                        }
                    }
                    post {
                        cleanup {
                            // clean up the working area but don't delete the download cache
                            dir ("htrace-noop") {
                                deleteDir()
                            }
                        }
                    }
                }
                stage ("test htrace drop in replacement") {
                    matrix {
                        agent {
                            label 'hbase'
                        }
                        axes {
                            axis {
                                name 'HBASE'
                                values '2.2.6', '2.3.3'
                            }
                            axis {
                                name 'HADOOP'
                                values '3.2.1', '2.10.0'
                            }
                        }
                        when {
                            allOf {
                                anyOf {
                                    expression { params.HBASE_VERSION == 'all' }
                                    expression { params.HBASE_VERSION == env.HBASE }
                                }
                                anyOf {
                                    expression { params.HADOOP_VERSION == 'all' }
                                    expression { params.HADOOP_VERSION == env.HADOOP }
                                }
                            }
                        }
                        stages {
                            stage ("test a specific hbase on a specific hadoop") {
                                steps {
                                    unstash 'scripts'
                                    unstash "hbase-${env.HBASE}"
                                    unstash "hadoop-${env.HADOOP}"
                                    sh '''#!/usr/bin/env bash
                                        set -eo
                                        set -x
                                        mkdir -p "hbase-${HBASE}.hadoop-${HADOOP}"
                                        ./tools/hbase_nightly_pseudo-distributed-test.sh \
                                                --single-process \
                                                --working-dir "hbase-${HBASE}.hadoop-${HADOOP}" \
                                                "hbase-${HBASE}" \
                                                "hadoop-${HADOOP}/bin/hadoop" \
                                                "hadoop-${HADOOP}"/share/hadoop/yarn/timelineservice \
                                                "hadoop-${HADOOP}"/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
                                                "hadoop-${HADOOP}"/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
                                                "hadoop-${HADOOP}"/bin/mapred \
                                    '''
                                }
                                post {
                                    failure {
                                        sh '''#!/usr/bin/env bash
                                            set -eo
                                            set -x
                                            find "hbase-${HBASE}.hadoop-${HADOOP}" \
                                                    "hbase-${HBASE}" "hadoop-${HADOOP}" \
                                                    -type d -name logs | \
                                                    xargs zip -r "hbase-${HBASE}.hadoop-${HADOOP}.logs.zip"
                                        '''
                                        archiveArtifacts artifacts: "hbase-${env.HBASE}.hadoop-${env.HADOOP}.logs.zip"
                                    }
                                    cleanup {
                                        deleteDir()
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    post {
        // Jenkins pipeline jobs fill slaves on PRs without this :(
        cleanup {
            script {
                sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
                    # See HADOOP-13951
                    chmod -R u+rxw "${WORKSPACE}"
                '''
            }
            // we purposefully don't do a top level workspace cleanup so that we can reuse downloads
        }
    }
}