build-aux/Jenkinsfile.full

#!groovy
//
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

build_and_test = '''
mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}
cd ${platform}
rm -rf build
mkdir build
cd build
tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
cd apache-couchdb-*
./configure --with-curl --spidermonkey-version ${sm_ver}
make check || (make build-report && false)
'''

make_packages = '''
cd ${platform}
git clone https://github.com/apache/couchdb-pkg
rm -rf couchdb
mkdir couchdb
cp ${WORKSPACE}/apache-couchdb-*.tar.gz couchdb
tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz -C couchdb
cd couchdb-pkg
make
'''

cleanup_and_save = '''
rm -rf ${WORKSPACE}/pkgs/${platform}
mkdir -p ${WORKSPACE}/pkgs/${platform}
mv ${WORKSPACE}/${platform}/rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
mv ${WORKSPACE}/${platform}/couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
'''

update_qemu = '''
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
'''

pipeline {

  // no top-level agent; agents must be declared for each stage
  agent none

  environment {
    COUCHAUTH = credentials('couchdb_vm2_couchdb')
    recipient = 'notifications@couchdb.apache.org'
    COUCHDB_IO_LOG_DIR = '/tmp/couchjslogs'
    // Following fix an issue with git <= 2.6.5 where no committer
    // name or email are present for reflog, required for git clone
    GIT_COMMITTER_NAME = 'Jenkins User'
    GIT_COMMITTER_EMAIL = 'couchdb@apache.org'
    // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
    // We need the jenkins user mapped inside of the image
    // npm config cache below deals with /home/jenkins not mapping correctly
    // inside the image
    DOCKER_ARGS = '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
  }

  options {
    buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
    // This fails the build immediately if any parallel step fails
    parallelsAlwaysFailFast()
    preserveStashes(buildCount: 10)
    timeout(time: 3, unit: 'HOURS')
    timestamps()
  }

  stages {
    stage('Build Release Tarball') {
      agent {
        docker {
          label 'docker'
          image 'couchdbdev/debian-stretch-erlang-20.3.8.25-1:latest'
          args "${DOCKER_ARGS}"
          alwaysPull true
        }
      }
      options {
        timeout(time: 15, unit: "MINUTES")
      }
      steps {
        sh '''
          set
          rm -rf apache-couchdb-*
          ./configure --with-curl
          make dist
          chmod -R a+w * .
        '''
      }
      post {
        success {
          stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball'
          archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true
        }
        cleanup {
          // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894
          sh 'rm -rf ${WORKSPACE}/*'
        }
      }
    } // stage Build Release Tarball

    // TODO Rework once Improved Docker Pipeline Engine is released
    // https://issues.jenkins-ci.org/browse/JENKINS-47962
    // https://issues.jenkins-ci.org/browse/JENKINS-48050

    stage('Test and Package') {

      options {
        skipDefaultCheckout()
        timeout(time: 90, unit: "MINUTES")
      }

      parallel {

        stage('FreeBSD') {
          agent {
            label 'freebsd'
          }
          steps {
            // deleteDir is OK here because we're not inside of a Docker container!
            deleteDir()
            unstash 'tarball'
            withEnv(['HOME='+pwd()]) {
              sh '''
                mkdir -p $COUCHDB_IO_LOG_DIR

                # Build CouchDB from tarball & test
                mkdir build
                cd build
                tar -xf $WORKSPACE/apache-couchdb-*.tar.gz
                cd apache-couchdb-*
                ./configure --with-curl
                gmake check || (build-aux/logfile-uploader.py && false)

                # No package build for FreeBSD at this time
              '''
            } // withEnv
          } // steps
          post {
            always {
              junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
            }
            cleanup {
              sh 'killall -9 beam.smp || true'
              sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
            }
          } // post
        } // stage FreeBSD

        stage('macOS') {
          agent {
            label 'macos'
          }
          steps {
            // deleteDir is OK here because we're not inside of a Docker container!
            deleteDir()
            unstash 'tarball'
            withEnv(['HOME='+pwd()]) {
              sh '''
                PATH=/usr/local/bin:$PATH
                export PATH
                mkdir -p $COUCHDB_IO_LOG_DIR

                # Build CouchDB from tarball & test
                mkdir build
                cd build
                tar -xzf $WORKSPACE/apache-couchdb-*.tar.gz
                cd apache-couchdb-*
                ./configure --with-curl --spidermonkey-version 60
                make check || (build-aux/logfile-uploader.py && false)

                # No package build for macOS at this time
              '''
            } // withEnv
          } // steps
          post {
            always {
              junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
            }
            cleanup {
              sh 'killall -9 beam.smp || true'
              sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
            }
          } // post
        } // stage macOS

        stage('CentOS 6') {
          agent {
            docker {
              image 'couchdbdev/centos-6-erlang-20.3.8.25-1:latest'
              label 'docker'
              args "${DOCKER_ARGS}"
              alwaysPull true
            }
          }
          environment {
            platform = 'centos6'
            sm_ver = '1.8.5'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('CentOS 7') {
          agent {
            docker {
              image 'couchdbdev/centos-7-erlang-20.3.8.25-1:latest'
              label 'docker'
              args "${DOCKER_ARGS}"
              alwaysPull true
            }
          }
          environment {
            platform = 'centos7'
            sm_ver = '1.8.5'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                unstash 'tarball'
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('CentOS 8') {
          agent {
            docker {
              image 'couchdbdev/centos-8-erlang-20.3.8.25-1:latest'
              label 'docker'
              args "${DOCKER_ARGS}"
              alwaysPull true
            }
          }
          environment {
            platform = 'centos8'
            sm_ver = '60'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                unstash 'tarball'
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('Ubuntu Xenial') {
          agent {
            docker {
              image 'couchdbdev/ubuntu-xenial-erlang-20.3.8.25-1:latest'
              label 'docker'
              args "${DOCKER_ARGS}"
              alwaysPull true
            }
          }
          environment {
            platform = 'xenial'
            sm_ver = '1.8.5'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('Ubuntu Bionic') {
          agent {
            docker {
              image 'couchdbdev/ubuntu-bionic-erlang-20.3.8.25-1:latest'
              label 'docker'
              alwaysPull true
              args "${DOCKER_ARGS}"
            }
          }
          environment {
            platform = 'bionic'
            sm_ver = '1.8.5'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('Ubuntu Focal') {
          agent {
            docker {
              image 'couchdbdev/ubuntu-focal-erlang-20.3.8.25-1:latest'
              label 'docker'
              alwaysPull true
              args "${DOCKER_ARGS}"
            }
          }
          environment {
            platform = 'focal'
            sm_ver = '68'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('Debian Stretch') {
          agent {
            docker {
              image 'couchdbdev/debian-stretch-erlang-20.3.8.25-1:latest'
              label 'docker'
              alwaysPull true
              args "${DOCKER_ARGS}"
            }
          }
          environment {
            platform = 'stretch'
            sm_ver = '1.8.5'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('Debian Buster amd64') {
          agent {
            docker {
              image 'couchdbdev/debian-buster-erlang-20.3.8.25-1:latest'
              label 'docker'
              alwaysPull true
              args "${DOCKER_ARGS}"
            }
          }
          environment {
            platform = 'buster'
            sm_ver = '60'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('Debian Buster arm64v8') {
          agent {
            docker {
              image 'couchdbdev/arm64v8-debian-buster-erlang-20.3.8.25-1:latest'
              label 'arm64v8'
              alwaysPull true
              args "${DOCKER_ARGS}"
            }
          }
          environment {
            platform = 'buster'
            sm_ver = '1.8.5'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

        stage('Debian Buster ppc64le') {
          agent {
            docker {
              image 'couchdbdev/ppc64le-debian-buster-erlang-20.3.8.25-1:latest'
              label 'ppc64le'
              alwaysPull true
              args "${DOCKER_ARGS}"
            }
          }
          environment {
            platform = 'buster'
            sm_ver = '60'
          }
          stages {
            stage('Build from tarball & test') {
              steps {
                unstash 'tarball'
                sh( script: build_and_test )
              }
              post {
                always {
                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
                }
              }
            }
            stage('Build CouchDB packages') {
              steps {
                sh( script: make_packages )
                sh( script: cleanup_and_save )
              }
              post {
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage

	/*
	 * Example of how to do a qemu-based run, please leave here
	 */

/*
        stage('Debian Buster arm64v8') {
          // the process is convoluted to ensure we have the latest qemu static binaries on the node first
          // before trying to run a foreign docker container type. Alternately ensuring the `update_qemu`
          // container is run on every Jenkins agent *after every restart of the Docker daemon* would work.
          agent {
            any {
            }
          }
          options {
            timeout(time: 120, unit: "MINUTES")
          }
          environment {
            platform = 'aarch64-debian-stretch'
            sm_ver = '60'
          }
          stages {
            stage('Install latest qemu binaries') {
              steps {
                sh( script: update_qemu )
              }
            }
            stage('Pull latest docker image') {
              steps {
                sh "docker pull couchdbdev/arm64v8-debian-buster-erlang-20.3.8.25-1:latest"
              }
            }
            stage('Build from tarball & test & packages') {
              steps {
                withDockerContainer(image: "couchdbdev/arm64v8-debian-buster-erlang-20.3.8.25-1:latest", args: "${DOCKER_ARGS}") {
                  unstash 'tarball'
                  withEnv(['MIX_HOME='+pwd(), 'HEX_HOME='+pwd()]) {
                    sh( script: build_and_test )
                    sh( script: make_packages )
                    sh( script: cleanup_and_save )
                  }
                }
              }
              post {
                always {
*/
//                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
/*
                }
                success {
                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
                }
              }
            }
          } // stages
          post {
            cleanup {
              sh 'rm -rf ${WORKSPACE}/*'
            }
          } // post
        } // stage
*/

      } // parallel
    } // stage "Test and Package"

    stage('Publish') {

      when {
        expression { return env.BRANCH_NAME ==~ /master|2.*.x|3.*.x|4.*.x|jenkins-.*/ }
      }

      agent {
        docker {
          image 'couchdbdev/debian-buster-erlang-20.3.8.25-1:latest'
          label 'docker'
          alwaysPull true
          args "${DOCKER_ARGS}"
        }
      }
      options {
        skipDefaultCheckout()
        timeout(time: 90, unit: "MINUTES")
      }

      steps {
        withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-key', keyFileVariable: 'KEY')]) {
          sh 'rm -rf ${WORKSPACE}/*'
          unstash 'tarball'
          unarchive mapping: ['pkgs/' : '.']

          echo 'Retrieving & cleaning current repo-nightly tree...'
          sh '''
            rsync -avz -e "ssh -o StrictHostKeyChecking=no -i $KEY" jenkins@repo-nightly.couchdb.org:/var/www/html/$BRANCH_NAME . || mkdir -p $BRANCH_NAME
            rm -rf $BRANCH_NAME/debian/* $BRANCH_NAME/el6/* $BRANCH_NAME/el7/* $BRANCH_NAME/el8/*
            mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el6 $BRANCH_NAME/el7 $BRANCH_NAME/el8 $BRANCH_NAME/source
            rsync -avz -e "ssh -o StrictHostKeyChecking=no -i $KEY" jenkins@repo-nightly.couchdb.org:/var/www/html/js .
          '''

          echo 'Building Debian repo...'
          sh '''
            git clone https://github.com/apache/couchdb-pkg
            cp js/debian-stretch/*.deb pkgs/stretch
            reprepro -b couchdb-pkg/repo includedeb stretch pkgs/stretch/*.deb
            cp js/debian-buster/*.deb pkgs/stretch
            reprepro -b couchdb-pkg/repo includedeb buster pkgs/buster/*.deb
            cp js/ubuntu-xenial/*.deb pkgs/xenial
            reprepro -b couchdb-pkg/repo includedeb xenial pkgs/xenial/*.deb
            cp js/ubuntu-bionic/*.deb pkgs/bionic
            reprepro -b couchdb-pkg/repo includedeb bionic pkgs/bionic/*.deb
            reprepro -b couchdb-pkg/repo includedeb focal pkgs/focal/*.deb
          '''

          echo 'Building CentOS repos...'
          sh '''
            cp js/centos-6/*rpm pkgs/centos6
            cp js/centos-7/*rpm pkgs/centos7
            cp js/centos-8/*rpm pkgs/centos8
            cd pkgs/centos6 && createrepo --database .
            cd ../centos7 && createrepo --database .
            cd ../centos8 && createrepo --database .
          '''

          echo 'Building tree to upload...'
          sh '''
            mv couchdb-pkg/repo/pool $BRANCH_NAME/debian
            mv couchdb-pkg/repo/dists $BRANCH_NAME/debian
            mv pkgs/centos6/* $BRANCH_NAME/el6
            mv pkgs/centos7/* $BRANCH_NAME/el7
            mv pkgs/centos8/* $BRANCH_NAME/el8
            mv apache-couchdb-*.tar.gz $BRANCH_NAME/source
            cd $BRANCH_NAME/source
            ls -1tr | head -n -10 | xargs -d '\n' rm -f --
            cd ../..
          '''

          echo 'rsyncing tree to repo-nightly...'
          sh '''
            rsync -avz --delete -e "ssh -o StrictHostKeyChecking=no -i $KEY" $BRANCH_NAME jenkins@repo-nightly.couchdb.org:/var/www/html
            rm -rf $BRANCH_NAME couchdb-pkg *.tar.gz
          '''
        } // withCredentials
      } // steps
    } // stage
  } // stages

  post {
    success {
      mail to: "${env.recipient}",
        replyTo: "${env.recipient}",
        subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}",
        body: "Yay, we passed. ${env.RUN_DISPLAY_URL}"
    }
    unstable {
      mail to: "${env.recipient}",
        replyTo: "${env.recipient}",
        subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}",
        body: "Eep! Build is unstable... ${env.RUN_DISPLAY_URL}"
    }
    failure {
      mail to: "${env.recipient}",
        replyTo: "${env.recipient}",
        subject: "[Jenkins] FAILURE: ${currentBuild.fullDisplayName}",
        body: "Boo, we failed. ${env.RUN_DISPLAY_URL}"
    }
  }

} // pipeline