| #!groovy |
| // |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| // use this file except in compliance with the License. You may obtain a copy of |
| // the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| // License for the specific language governing permissions and limitations under |
| // the License. |
| |
| // Erlang version embedded in binary packages |
| ERLANG_VERSION = '25.3.2.18' |
| |
| // Erlang version used for rebar in release process. CouchDB will not build from |
| // the release tarball on Erlang versions older than this |
| MINIMUM_ERLANG_VERSION = '25.3.2.18' |
| |
| // We create parallel build / test / package stages for each OS using the metadata |
| // in this map. Adding a new OS should ideally only involve adding a new entry here. |
| meta = [ |
| 'centos8': [ |
| name: 'CentOS 8', |
| spidermonkey_vsn: '60', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'centos9': [ |
| name: 'CentOS 9', |
| spidermonkey_vsn: '78', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-centos:9-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'focal': [ |
| name: 'Ubuntu 20.04', |
| spidermonkey_vsn: '68', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'jammy': [ |
| name: 'Ubuntu 22.04', |
| spidermonkey_vsn: '91', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'noble': [ |
| name: 'Ubuntu 24.04', |
| spidermonkey_vsn: '115', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'bookworm-ppc64': [ |
| name: 'Debian POWER', |
| spidermonkey_vsn: '78', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", |
| node_label: 'ppc64le' |
| ], |
| |
| 'bookworm-s390x': [ |
| name: 'Debian s390x', |
| spidermonkey_vsn: '78', |
| with_nouveau: true, |
| image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}", |
| node_label: 's390x' |
| ], |
| |
| 'bullseye': [ |
| name: 'Debian x86_64', |
| spidermonkey_vsn: '78', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'bookworm': [ |
| name: 'Debian x86_64', |
| spidermonkey_vsn: '78', |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'bookworm-quickjs': [ |
| name: 'Debian 12 with QuickJS', |
| disable_spidermonkey: true, |
| with_nouveau: true, |
| with_clouseau: true, |
| image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}" |
| ], |
| |
| 'freebsd-x86_64': [ |
| name: 'FreeBSD x86_64', |
| spidermonkey_vsn: '91', |
| with_clouseau: false, |
| gnu_make: 'gmake' |
| ], |
| |
| // Spidermonkey 91 has issues on ARM64 FreeBSD |
| // use QuickJS for now |
| 'freebsd-arm64': [ |
| name: 'FreeBSD ARM64 QuickJS', |
| disable_spidermonkey: true, |
| with_clouseau: false, |
| gnu_make: 'gmake' |
| ], |
| |
| // Excluding clouseau for macos. This stopped working as |
| // of https://github.com/apache/couchdb/pull/5180 (bump to 2.23.1) |
| // but works for the other workers. |
| 'macos': [ |
| name: 'macOS', |
| spidermonkey_vsn: '128', |
| with_nouveau: false, |
| with_clouseau: false, |
| gnu_make: 'make' |
| ], |
| |
| 'win2022': [ |
| name: 'Windows 2022', |
| spidermonkey_vsn: '128', |
| node_label: 'win' |
| ] |
| ] |
| |
| def String configure(config) { |
| if (config.disable_spidermonkey) { |
| result = "./configure --skip-deps --disable-spidermonkey" |
| } else { |
| result = "./configure --skip-deps --spidermonkey-version ${config.spidermonkey_vsn}" |
| } |
| if (config.with_nouveau) { |
| result += " --with-nouveau" |
| } |
| if (config.with_clouseau) { |
| result += " --with-clouseau" |
| } |
| return result |
| } |
| |
| // Credit to https://stackoverflow.com/a/69222555 for this technique. |
| // We can use the scripted pipeline syntax to dynamically generate stages, |
| // and inject them into a map that we pass to the `parallel` step in a script. |
| // While the scripting approach is very flexible, it's not able to use some |
| // functionality specific to Declarative Pipelines, like the `agent` and `post` |
| // directives, so you'll see alternatives like try-catch-finally used for flow |
| // control and the nested `node` and `docker` blocks in the container stage to |
| // configure the worker environment. |
| |
| // Returns a build stage suitable for non-containerized environments (currently |
| // macOS and FreeBSD). Coincidentally we do not currently support automated |
| // package generation on these platforms. This method in invoked when we create |
| // `parallelStagesMap` below. |
| def generateNativeStage(platform) { |
| return { |
| stage(platform) { |
| node(platform) { |
| timeout(time: 90, unit: "MINUTES") { |
| // Steps to configure and build CouchDB on *nix platforms |
| if (isUnix()) { |
| try { |
| // deleteDir is OK here because we're not inside of a Docker container! |
| deleteDir() |
| unstash 'tarball' |
| withEnv([ |
| 'HOME='+pwd(), |
| 'PATH+USRLOCAL=/usr/local/bin', |
| 'MAKE='+meta[platform].gnu_make |
| ]) { |
| sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' ) |
| sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) |
| dir( "${platform}/build" ) { |
| sh "${configure(meta[platform])}" |
| sh '$MAKE' |
| sh '$MAKE eunit' |
| sh '$MAKE elixir' |
| sh '$MAKE elixir-search' |
| sh '$MAKE mango-test' |
| sh '$MAKE weatherreport-test' |
| sh '$MAKE nouveau-test' |
| } |
| } |
| } |
| catch (err) { |
| sh 'ls -l ${WORKSPACE}' |
| withEnv([ |
| 'HOME='+pwd(), |
| 'PATH+USRLOCAL=/usr/local/bin', |
| 'MAKE='+meta[platform].gnu_make |
| ]) { |
| dir( "${platform}/build" ) { |
| sh 'ls -l' |
| sh '${MAKE} build-report' |
| } |
| } |
| error("Build step failed with error: ${err.getMessage()}") |
| } |
| finally { |
| junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' |
| sh 'killall -9 beam.smp || true' |
| sh 'rm -rf ${WORKSPACE}/*' |
| } |
| } else { |
| //steps to configure and build CouchDB on Windows platforms |
| stage("${meta[platform].name} - build & test") { |
| try { |
| // deleteDir is OK here because we're not inside of a Docker container! |
| deleteDir() |
| unstash 'tarball' |
| powershell( script: "git clone https://github.com/apache/couchdb-glazier", label: 'Cloning couchdb-glazier repository' ) |
| powershell( script: "New-Item -ItemType Directory -Path '${platform}/build' -Force", label: 'Create build directories' ) |
| powershell( script: "tar -xf (Get-Item apache-couchdb-*.tar.gz) -C '${platform}/build' --strip-components=1", label: 'Unpack release' ) |
| dir( "${platform}/build" ) { |
| powershell( script: """ |
| .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 |
| .\\configure.ps1 -SkipDeps -WithNouveau -SpiderMonkeyVersion ${meta[platform].spidermonkey_vsn} |
| Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' |
| make -f Makefile.win release |
| """, label: 'Configure and Build') |
| |
| //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win eunit", label: 'EUnit tests') |
| //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win elixir", label: 'Elixir tests') |
| |
| powershell( script: '& .\\..\\..\\couchdb-glazier\\bin\\shell.ps1; Write-Host "NOT AVAILABLE: make -f Makefile.win elixir-search"', label: 'N/A Clouseau tests') |
| |
| powershell( script: """ |
| .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 |
| Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' |
| make -f Makefile.win mango-test |
| """, label: 'Mango tests') |
| |
| powershell( script: '.\\..\\..\\couchdb-glazier\\bin\\shell.ps1; Write-Host "NOT AVAILABLE: make -f Makefile.win weatherreport-test"', label: 'N/A Weatherreport tests') |
| |
| // temporary exclude - random flaky tests on Windows |
| //powershell( script: """ |
| // .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 |
| // Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' |
| // make -f Makefile.win nouveau-test |
| //""", label: 'Nouveau tests') |
| } |
| |
| powershell( script: """ |
| .\\couchdb-glazier\\bin\\shell.ps1 |
| .\\couchdb-glazier\\bin\\build_installer.ps1 -Path '${platform}/build' -IncludeGitSha -DisableICEChecks |
| """, label: 'Build Windows Installer file') |
| |
| archiveArtifacts artifacts: '*.msi', fingerprint: true, onlyIfSuccessful: true |
| } |
| catch (err) { |
| powershell( script: "Get-ChildItem ${WORKSPACE}") |
| dir( "${platform}/build" ) { |
| powershell( script: '.\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win build-report') |
| powershell( script: 'Get-Content test-results.log') |
| } |
| error("Build step failed with error: ${err.getMessage()}") |
| } |
| finally { |
| powershell( script: 'Get-ChildItem') |
| powershell( script: "Remove-Item -Path '${WORKSPACE}\\*' -Force -Recurse -ErrorAction SilentlyContinue") |
| powershell( script: 'Get-ChildItem') |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| // Returns a build stage suitable for container-based deployments. This method |
| // is invoked when we create the `parallelStagesMap` in the pipeline below. |
| def generateContainerStage(platform) { |
| return { |
| // Important: the stage name here must match the parallelStagesMap key for the |
| // Jenkins UI to render the pipeline stages correctly. Don't ask why. -APK |
| stage(platform) { |
| node(meta[platform].get('node_label', 'docker')) { |
| docker.withRegistry('https://docker.io/', 'dockerhub_creds') { |
| docker.image(meta[platform].image).inside("${DOCKER_ARGS}") { |
| timeout(time: 90, unit: "MINUTES") { |
| stage("${meta[platform].name} - build & test") { |
| try { |
| sh( script: "rm -rf ${platform} apache-couchdb-*", label: 'Clean workspace' ) |
| unstash 'tarball' |
| sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' ) |
| sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) |
| dir( "${platform}/build" ) { |
| sh "${configure(meta[platform])}" |
| sh 'make' |
| retry(3) {sh 'make eunit'} |
| retry(3) {sh 'make elixir'} |
| retry(3) {sh 'make elixir-search'} |
| retry(3) {sh 'make mango-test'} |
| retry(3) {sh 'make weatherreport-test'} |
| retry(3) {sh 'make nouveau-test'} |
| } |
| } |
| catch (err) { |
| sh 'ls -l ${WORKSPACE}' |
| dir( "${platform}/build" ) { |
| sh 'ls -l' |
| sh 'make build-report' |
| } |
| error("Build step failed with error: ${err.getMessage()}") |
| } |
| finally { |
| junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' |
| sh 'rm -rf ${WORKSPACE}/*' |
| } |
| } |
| |
| stage("${meta[platform].name} - package") { |
| try { |
| unstash 'tarball' |
| sh( script: "mkdir -p ${platform}/couchdb", label: 'Create build directory' ) |
| sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/couchdb", label: 'Unpack release' ) |
| sh( script: "cd ${platform} && git clone https://github.com/apache/couchdb-pkg", label: 'Clone packaging helper repo' ) |
| dir( "${platform}/couchdb-pkg" ) { |
| sh( script: 'make', label: 'Build packages' ) |
| } |
| sh( label: 'Stage package artifacts for archival', script: """ |
| rm -rf pkgs/${platform} |
| mkdir -p pkgs/${platform} |
| mv ${platform}/rpmbuild/RPMS/\$(arch)/*rpm pkgs/${platform} || true |
| mv ${platform}/couchdb/*.deb pkgs/${platform} || true |
| """ ) |
| archiveArtifacts artifacts: 'pkgs/**', fingerprint: true, onlyIfSuccessful: true |
| } |
| catch (err) { |
| sh 'ls -l ${WORKSPACE}' |
| error("Build step failed with error: ${err.getMessage()}") |
| } |
| finally { |
| sh 'rm -rf ${WORKSPACE}/*' |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| // Finally we have the actual Pipeline. It's mostly a Declarative Pipeline, |
| // except for the 'Test and Package' stage where we use the `script` step as an |
| // "escape hatch" to dynamically generate a set of parallel stages to execute. |
| pipeline { |
| |
| // no top-level agent; agents must be declared for each stage |
| agent none |
| |
| environment { |
| // Following fix an issue with git <= 2.6.5 where no committer |
| // name or email are present for reflog, required for git clone |
| GIT_COMMITTER_NAME = 'Jenkins User' |
| GIT_COMMITTER_EMAIL = 'couchdb@apache.org' |
| // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64 |
| // We need the jenkins user mapped inside of the image |
| // npm config cache below deals with /home/jenkins not mapping correctly |
| // inside the image |
| DOCKER_ARGS = '-e npm_config_cache=/home/jenkins/.npm -e HOME=. -e MIX_HOME=/home/jenkins/.mix -e HEX_HOME=/home/jenkins/.hex -e PIP_CACHE_DIR=/home/jenkins/.cache/pip -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group -v /home/jenkins/.gradle:/home/jenkins/.gradle:rw,z -v /home/jenkins/.hex:/home/jenkins/.hex:rw,z -v /home/jenkins/.npm:/home/jenkins/.npm:rw,z -v /home/jenkins/.cache/pip:/home/jenkins/.cache/pip:rw,z -v /home/jenkins/.mix:/home/jenkins/.mix:rw,z' |
| } |
| |
| options { |
| buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) |
| preserveStashes(buildCount: 10) |
| timeout(time: 3, unit: 'HOURS') |
| timestamps() |
| } |
| |
| stages { |
| stage('Build Release Tarball') { |
| agent { |
| docker { |
| label 'docker' |
| image "apache/couchdbci-debian:bookworm-erlang-${MINIMUM_ERLANG_VERSION}" |
| args "${DOCKER_ARGS}" |
| registryUrl 'https://docker.io/' |
| registryCredentialsId 'dockerhub_creds' |
| } |
| } |
| steps { |
| timeout(time: 15, unit: "MINUTES") { |
| sh (script: 'rm -rf apache-couchdb-*', label: 'Clean workspace of any previous release artifacts' ) |
| sh "./configure --spidermonkey-version 78 --with-nouveau" |
| sh 'make dist' |
| } |
| } |
| post { |
| success { |
| stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball' |
| archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true |
| } |
| failure { |
| sh 'ls -l ${WORKSPACE}' |
| } |
| cleanup { |
| // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 |
| sh 'rm -rf ${WORKSPACE}/*' |
| } |
| } |
| } // stage Build Release Tarball |
| |
| stage('Test and Package') { |
| steps { |
| script { |
| // Including failFast: true in map fails the build immediately if any parallel step fails |
| parallelStagesMap = meta.collectEntries( [failFast: false] ) { key, values -> |
| if (values.image) { |
| ["${key}": generateContainerStage(key)] |
| } |
| else { |
| ["${key}": generateNativeStage(key)] |
| } |
| } |
| parallel parallelStagesMap |
| } |
| } |
| } |
| |
| stage('Publish') { |
| |
| when { |
| expression { return env.BRANCH_NAME ==~ /main|2.*.x|3.*.x|4.*.x|jenkins-.*/ } |
| } |
| |
| agent { |
| docker { |
| image "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" |
| label 'docker' |
| args "${DOCKER_ARGS}" |
| registryUrl 'https://docker.io/' |
| registryCredentialsId 'dockerhub_creds' |
| } |
| } |
| options { |
| skipDefaultCheckout() |
| timeout(time: 90, unit: "MINUTES") |
| } |
| |
| steps { |
| sh 'rm -rf ${WORKSPACE}/*' |
| unstash 'tarball' |
| unarchive mapping: ['pkgs/' : '.'] |
| |
| sh( label: 'Setup repo dirs', script: ''' |
| mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el8 $BRANCH_NAME/el9 $BRANCH_NAME/source |
| git clone https://github.com/apache/couchdb-pkg |
| ''' ) |
| |
| sh( label: 'Build Debian repo', script: ''' |
| for plat in bullseye bookworm focal |
| do |
| reprepro -b couchdb-pkg/repo includedeb $plat pkgs/$plat/*.deb |
| done |
| ''' ) |
| |
| sh( label: 'Build CentOS 8', script: ''' |
| (cd pkgs/centos8 && createrepo_c --database .) |
| ''' ) |
| |
| sh( label: 'Build CentOS 9', script: ''' |
| (cd pkgs/centos9 && createrepo_c --database .) |
| ''' ) |
| |
| sh( label: 'Build unified repo', script: ''' |
| mv couchdb-pkg/repo/pool $BRANCH_NAME/debian |
| mv couchdb-pkg/repo/dists $BRANCH_NAME/debian |
| mv pkgs/centos8/* $BRANCH_NAME/el8 |
| mv pkgs/centos9/* $BRANCH_NAME/el9 |
| mv apache-couchdb-*.tar.gz $BRANCH_NAME/source |
| cd $BRANCH_NAME/source |
| ls -1tr | head -n -10 | xargs -d '\n' rm -f -- |
| cd ../.. |
| ''' ) |
| } // steps |
| } // stage |
| } // stages |
| |
| post { |
| success { |
| mail to: 'notifications@couchdb.apache.org', |
| replyTo: 'notifications@couchdb.apache.org', |
| subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}", |
| body: "Yay, we passed. ${env.RUN_DISPLAY_URL}" |
| } |
| unstable { |
| mail to: 'notifications@couchdb.apache.org', |
| replyTo: 'notifications@couchdb.apache.org', |
| subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}", |
| body: "Eep! Build is unstable... ${env.RUN_DISPLAY_URL}" |
| } |
| failure { |
| mail to: 'notifications@couchdb.apache.org', |
| replyTo: 'notifications@couchdb.apache.org', |
| subject: "[Jenkins] FAILURE: ${currentBuild.fullDisplayName}", |
| body: "Boo, we failed. ${env.RUN_DISPLAY_URL}" |
| } |
| } |
| |
| } // pipeline |
