Merge branch 'main' into update-erlfmt
diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full
index 6cf5e2b..328812c 100644
--- a/build-aux/Jenkinsfile.full
+++ b/build-aux/Jenkinsfile.full
@@ -13,40 +13,227 @@
// License for the specific language governing permissions and limitations under
// the License.
-build_and_test = '''
-mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}
-cd ${platform}
-rm -rf build
-mkdir build
-cd build
-tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
-cd apache-couchdb-*
-./configure --spidermonkey-version ${sm_ver}
-make check || (make build-report && false)
-'''
+// Erlang version embedded in binary packages
+ERLANG_VERSION = '24.2'
-make_packages = '''
-cd ${platform}
-git clone https://github.com/apache/couchdb-pkg
-rm -rf couchdb
-mkdir couchdb
-cp ${WORKSPACE}/apache-couchdb-*.tar.gz couchdb
-tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz -C couchdb
-cd couchdb-pkg
-make
-'''
+// Erlang version used for rebar in release process. CouchDB will not build from
+// the release tarball on Erlang versions older than this
+MINIMUM_ERLANG_VERSION = '21'
-cleanup_and_save = '''
-rm -rf ${WORKSPACE}/pkgs/${platform}
-mkdir -p ${WORKSPACE}/pkgs/${platform}
-mv ${WORKSPACE}/${platform}/rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
-mv ${WORKSPACE}/${platform}/couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
-'''
+// We create parallel build / test / package stages for each OS using the metadata
+// in this map. Adding a new OS should ideally only involve adding a new entry here.
+meta = [
+ 'centos7': [
+ name: 'CentOS 7',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}"
+ ],
-update_qemu = '''
-docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-'''
+ 'centos8': [
+ name: 'CentOS 8',
+ spidermonkey_vsn: '60',
+ image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}"
+ ],
+ 'bionic': [
+ name: 'Ubuntu 18.04',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'focal': [
+ name: 'Ubuntu 20.04',
+ spidermonkey_vsn: '68',
+ image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'stretch': [
+ name: 'Debian 9',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-debian:stretch-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'buster': [
+ name: 'Debian 10',
+ spidermonkey_vsn: '60',
+ image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}"
+ ],
+
+ // - Removed 2020.09.15 - VMs are offline
+
+ // 'buster-arm64': [
+ // name: 'Debian 10 ARM'
+ // spidermonkey_vsn: '60',
+ // image: "apache/couchdbci-debian:arm64v8-buster-erlang-${ERLANG_VERSION}",
+ // node_label: 'arm64v8'
+ // ],
+
+ // 'buster-ppc64': [
+ // name: 'Debian 10 POWER'
+ // spidermonkey_vsn: '60',
+ // image: "apache/couchdbci-debian:pp64le-buster-erlang-${ERLANG_VERSION}",
+ // node_label: 'ppc64le'
+ // ],
+
+ 'bullseye': [
+ name: 'Debian 11',
+ spidermonkey_vsn: '78',
+ image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'freebsd': [
+ name: 'FreeBSD',
+ spidermonkey_vsn: '1.8.5',
+ gnu_make: 'gmake'
+ ],
+
+ 'macos': [
+ name: 'macOS',
+ spidermonkey_vsn: '60',
+ gnu_make: 'make'
+ ]
+]
+
+// Credit to https://stackoverflow.com/a/69222555 for this technique.
+// We can use the scripted pipeline syntax to dynamically generate stages,
+// and inject them into a map that we pass to the `parallel` step in a script.
+// While the scripting approach is very flexible, it's not able to use some
+// functionality specific to Declarative Pipelines, like the `agent` and `post`
+// directives, so you'll see alternatives like try-catch-finally used for flow
+// control and the nested `node` and `docker` blocks in the container stage to
+// configure the worker environment.
+
+// Returns a build stage suitable for non-containerized environments (currently
+// macOS and FreeBSD). Coincidentally we do not currently support automated
+// package generation on these platforms. This method in invoked when we create
+// `parallelStagesMap` below.
+def generateNativeStage(platform) {
+ return {
+ stage(platform) {
+ node(platform) {
+ timeout(time: 90, unit: "MINUTES") {
+ try {
+ // deleteDir is OK here because we're not inside of a Docker container!
+ deleteDir()
+ unstash 'tarball'
+ withEnv([
+ 'HOME='+pwd(),
+ 'PATH+USRLOCAL=/usr/local/bin',
+ 'MAKE='+meta[platform].gnu_make
+ ]) {
+ sh( script: "mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}/build", label: 'Create build directories' )
+ sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' )
+ dir( "${platform}/build" ) {
+ sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+ sh '$MAKE'
+ sh '$MAKE eunit'
+ sh '$MAKE elixir-suite'
+ sh '$MAKE exunit'
+ sh '$MAKE mango-test'
+ }
+ }
+ }
+ catch (err) {
+ sh 'ls -l ${WORKSPACE}'
+ withEnv([
+ 'HOME='+pwd(),
+ 'PATH+USRLOCAL=/usr/local/bin',
+ 'MAKE='+meta[platform].gnu_make
+ ]) {
+ dir( "${platform}/build" ) {
+ sh 'ls -l'
+ sh '${MAKE} build-report'
+ }
+ }
+ error("Build step failed with error: ${err.getMessage()}")
+ }
+ finally {
+ junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+ sh 'killall -9 beam.smp || true'
+ sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR}'
+ }
+ }
+ }
+ }
+ }
+}
+
+// Returns a build stage suitable for container-based deployments. This method
+// is invoked when we create the `parallelStagesMap` in the pipeline below.
+def generateContainerStage(platform) {
+ return {
+ // Important: the stage name here must match the parallelStagesMap key for the
+ // Jenkins UI to render the pipeline stages correctly. Don't ask why. -APK
+ stage(platform) {
+ node(meta[platform].get('node_label', 'docker')) {
+ docker.withRegistry('https://docker.io/', 'dockerhub_creds') {
+ docker.image(meta[platform].image).inside("${DOCKER_ARGS}") {
+ timeout(time: 90, unit: "MINUTES") {
+ stage("${meta[platform].name} - build & test") {
+ try {
+ sh( script: "rm -rf ${platform} apache-couchdb-*", label: 'Clean workspace' )
+ unstash 'tarball'
+ sh( script: "mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}/build", label: 'Create build directories' )
+ sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' )
+ dir( "${platform}/build" ) {
+ sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+ sh 'make'
+ sh 'make eunit'
+ sh 'make elixir-suite'
+ sh 'make exunit'
+ sh 'make mango-test'
+ }
+ }
+ catch (err) {
+ sh 'ls -l ${WORKSPACE}'
+ dir( "${platform}/build" ) {
+ sh 'ls -l'
+ sh 'make build-report'
+ }
+ error("Build step failed with error: ${err.getMessage()}")
+ }
+ finally {
+ junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+ sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR}'
+ }
+ }
+
+ stage("${meta[platform].name} - package") {
+ try {
+ unstash 'tarball'
+ sh( script: "mkdir -p ${platform}/couchdb", label: 'Create build directory' )
+ sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/couchdb", label: 'Unpack release' )
+ sh( script: "cd ${platform} && git clone https://github.com/apache/couchdb-pkg", label: 'Clone packaging helper repo' )
+ dir( "${platform}/couchdb-pkg" ) {
+ sh( script: 'make', label: 'Build packages' )
+ }
+ sh( label: 'Stage package artifacts for archival', script: """
+ rm -rf pkgs/${platform}
+ mkdir -p pkgs/${platform}
+ mv ${platform}/rpmbuild/RPMS/\$(arch)/*rpm pkgs/${platform} || true
+ mv ${platform}/couchdb/*.deb pkgs/${platform} || true
+ """ )
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true, onlyIfSuccessful: true
+ }
+ catch (err) {
+ sh 'ls -l ${WORKSPACE}'
+ error("Build step failed with error: ${err.getMessage()}")
+ }
+ finally {
+ sh 'rm -rf ${WORKSPACE}/*'
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+// Finally we have the actual Pipeline. It's mostly a Declarative Pipeline,
+// except for the 'Test and Package' stage where we use the `script` step as an
+// "escape hatch" to dynamically generate a set of parallel stages to execute.
pipeline {
// no top-level agent; agents must be declared for each stage
@@ -68,8 +255,6 @@
options {
buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
- // This fails the build immediately if any parallel step fails
- parallelsAlwaysFailFast()
preserveStashes(buildCount: 10)
timeout(time: 3, unit: 'HOURS')
timestamps()
@@ -80,30 +265,33 @@
agent {
docker {
label 'docker'
- image 'apache/couchdbci-debian:buster-erlang-21.3.8.17-1'
+ image "apache/couchdbci-debian:erlang-${MINIMUM_ERLANG_VERSION}"
args "${DOCKER_ARGS}"
registryUrl 'https://docker.io/'
registryCredentialsId 'dockerhub_creds'
}
}
- options {
- timeout(time: 15, unit: "MINUTES")
+ environment {
+ // TODO find a way to avoid setting this explicitly
+ spidermonkey = '78'
}
steps {
- sh '''
- set
- rm -rf apache-couchdb-*
- ./configure
- make erlfmt-check
- make dist
- chmod -R a+w * .
- '''
+ timeout(time: 15, unit: "MINUTES") {
+ sh (script: 'rm -rf apache-couchdb-*', label: 'Clean workspace of any previous release artifacts' )
+ sh "./configure --spidermonkey-version ${spidermonkey}"
+ sh 'make erlfmt-check'
+ sh 'make elixir-check-formatted'
+ sh 'make dist'
+ }
}
post {
success {
stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball'
archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true
}
+ failure {
+ sh 'ls -l ${WORKSPACE}'
+ }
cleanup {
// UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894
sh 'rm -rf ${WORKSPACE}/*'
@@ -111,457 +299,22 @@
}
} // stage Build Release Tarball
- // TODO Rework once Improved Docker Pipeline Engine is released
- // https://issues.jenkins-ci.org/browse/JENKINS-47962
- // https://issues.jenkins-ci.org/browse/JENKINS-48050
-
stage('Test and Package') {
-
- options {
- skipDefaultCheckout()
- timeout(time: 90, unit: "MINUTES")
+ steps {
+ script {
+ // Including failFast: true in map fails the build immediately if any parallel step fails
+ parallelStagesMap = meta.collectEntries( [failFast: false] ) { key, values ->
+ if (values.image) {
+ ["${key}": generateContainerStage(key)]
+ }
+ else {
+ ["${key}": generateNativeStage(key)]
+ }
+ }
+ parallel parallelStagesMap
+ }
}
-
- parallel {
-
- stage('FreeBSD') {
- agent {
- label 'freebsd'
- }
- steps {
- // deleteDir is OK here because we're not inside of a Docker container!
- deleteDir()
- unstash 'tarball'
- withEnv(['HOME='+pwd()]) {
- sh '''
- mkdir -p $COUCHDB_IO_LOG_DIR
-
- # Build CouchDB from tarball & test
- mkdir build
- cd build
- tar -xf $WORKSPACE/apache-couchdb-*.tar.gz
- cd apache-couchdb-*
- ./configure
- gmake check || (build-aux/logfile-uploader.py && false)
-
- # No package build for FreeBSD at this time
- '''
- } // withEnv
- } // steps
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- cleanup {
- sh 'killall -9 beam.smp || true'
- sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
- }
- } // post
- } // stage FreeBSD
-
- stage('macOS') {
- agent {
- label 'macos'
- }
- steps {
- // deleteDir is OK here because we're not inside of a Docker container!
- deleteDir()
- unstash 'tarball'
- withEnv(['HOME='+pwd()]) {
- sh '''
- PATH=/usr/local/bin:$PATH
- export PATH
- mkdir -p $COUCHDB_IO_LOG_DIR
-
- # Build CouchDB from tarball & test
- mkdir build
- cd build
- tar -xzf $WORKSPACE/apache-couchdb-*.tar.gz
- cd apache-couchdb-*
- ./configure --spidermonkey-version 60
- make check || (build-aux/logfile-uploader.py && false)
-
- # No package build for macOS at this time
- '''
- } // withEnv
- } // steps
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- cleanup {
- sh 'killall -9 beam.smp || true'
- sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
- }
- } // post
- } // stage macOS
-
- stage('CentOS 7') {
- agent {
- docker {
- image 'apache/couchdbci-centos:7-erlang-21.3.8.17-1'
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
- environment {
- platform = 'centos7'
- sm_ver = '1.8.5'
- }
- stages {
- stage('Build from tarball & test') {
- steps {
- unstash 'tarball'
- sh( script: build_and_test )
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- }
- }
- stage('Build CouchDB packages') {
- steps {
- unstash 'tarball'
- sh( script: make_packages )
- sh( script: cleanup_and_save )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- } // stages
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // stage
-
- stage('CentOS 8') {
- agent {
- docker {
- image 'apache/couchdbci-centos:8-erlang-21.3.8.17-1'
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
- environment {
- platform = 'centos8'
- sm_ver = '60'
- }
- stages {
- stage('Build from tarball & test') {
- steps {
- unstash 'tarball'
- sh( script: build_and_test )
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- }
- }
- stage('Build CouchDB packages') {
- steps {
- unstash 'tarball'
- sh( script: make_packages )
- sh( script: cleanup_and_save )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- } // stages
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // stage
-
- stage('Ubuntu Bionic') {
- agent {
- docker {
- image 'apache/couchdbci-ubuntu:bionic-erlang-21.3.8.17-1'
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
- environment {
- platform = 'bionic'
- sm_ver = '1.8.5'
- }
- stages {
- stage('Build from tarball & test') {
- steps {
- unstash 'tarball'
- sh( script: build_and_test )
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- }
- }
- stage('Build CouchDB packages') {
- steps {
- sh( script: make_packages )
- sh( script: cleanup_and_save )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- } // stages
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // stage
-
- stage('Ubuntu Focal') {
- agent {
- docker {
- image 'apache/couchdbci-ubuntu:focal-erlang-21.3.8.17-1'
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
- environment {
- platform = 'focal'
- sm_ver = '68'
- }
- stages {
- stage('Build from tarball & test') {
- steps {
- unstash 'tarball'
- sh( script: build_and_test )
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- }
- }
- stage('Build CouchDB packages') {
- steps {
- sh( script: make_packages )
- sh( script: cleanup_and_save )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- } // stages
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // stage
-
- stage('Debian Stretch') {
- agent {
- docker {
- image 'apache/couchdbci-debian:stretch-erlang-21.3.8.17-1'
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
- environment {
- platform = 'stretch'
- sm_ver = '1.8.5'
- }
- stages {
- stage('Build from tarball & test') {
- steps {
- unstash 'tarball'
- sh( script: build_and_test )
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- }
- }
- stage('Build CouchDB packages') {
- steps {
- sh( script: make_packages )
- sh( script: cleanup_and_save )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- } // stages
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // stage
-
- stage('Debian Buster amd64') {
- agent {
- docker {
- image 'apache/couchdbci-debian:buster-erlang-21.3.8.17-1'
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
- environment {
- platform = 'buster'
- sm_ver = '60'
- }
- stages {
- stage('Build from tarball & test') {
- steps {
- unstash 'tarball'
- sh( script: build_and_test )
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- }
- }
- stage('Build CouchDB packages') {
- steps {
- sh( script: make_packages )
- sh( script: cleanup_and_save )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- } // stages
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // stage
-
- stage('Debian Buster arm64v8') {
- when { expression { return false } }
- agent {
- docker {
- image 'apache/couchdbci-debian:arm64v8-buster-erlang-21.3.8.17-1'
- label 'arm64v8'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
- environment {
- platform = 'buster'
- sm_ver = '1.8.5'
- }
- stages {
- stage('Build from tarball & test') {
- steps {
- unstash 'tarball'
- sh( script: build_and_test )
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- }
- }
- stage('Build CouchDB packages') {
- steps {
- sh( script: make_packages )
- sh( script: cleanup_and_save )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- } // stages
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // stage
-
-/*
- - Removed 2020.09.15 - VMs are offline
-*/
-
-// stage('Debian Buster ppc64le') {
-// agent {
-// docker {
-// image 'apache/couchdbci-debian:ppc64le-buster-erlang-21.3.8.17-1'
-// label 'ppc64le'
-// args "${DOCKER_ARGS}"
-// registryUrl 'https://docker.io/'
-// registryCredentialsId 'dockerhub_creds'
-// }
-// }
-// environment {
-// platform = 'buster'
-// sm_ver = '60'
-// }
-// stages {
-// stage('Build from tarball & test') {
-// steps {
-// unstash 'tarball'
-// sh( script: build_and_test )
-// }
-// post {
-// always {
-// junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
-// }
-// }
-// }
-// stage('Build CouchDB packages') {
-// steps {
-// sh( script: make_packages )
-// sh( script: cleanup_and_save )
-// }
-// post {
-// success {
-// archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
-// }
-// }
-// }
-// } // stages
-// post {
-// cleanup {
-// sh 'rm -rf ${WORKSPACE}/*'
-// }
-// } // post
-// } // stage
+ }
/*
* Example of how to do a qemu-based run, please leave here
@@ -586,17 +339,17 @@
stages {
stage('Install latest qemu binaries') {
steps {
- sh( script: update_qemu )
+ sh( script: 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes' )
}
}
stage('Pull latest docker image') {
steps {
- sh "docker pull apache/couchdbci-debian:arm64v8-buster-erlang-21.3.8.17-1"
+ sh "docker pull apache/couchdbci-debian:arm64v8-buster-erlang-${ERLANG_VERSION}"
}
}
stage('Build from tarball & test & packages') {
steps {
- withDockerContainer(image: "apache/couchdbci-debian:arm64v8-buster-erlang-21.3.8.17-1", args: "${DOCKER_ARGS}") {
+ withDockerContainer(image: "apache/couchdbci-debian:arm64v8-buster-erlang-${ERLANG_VERSION}", args: "${DOCKER_ARGS}") {
unstash 'tarball'
withEnv(['MIX_HOME='+pwd(), 'HEX_HOME='+pwd()]) {
sh( script: build_and_test )
@@ -625,9 +378,6 @@
} // stage
*/
- } // parallel
- } // stage "Test and Package"
-
stage('Publish') {
when {
@@ -636,7 +386,7 @@
agent {
docker {
- image 'apache/couchdbci-debian:buster-erlang-21.3.8.17-1'
+ image "apache/couchdbci-debian:erlang-${ERLANG_VERSION}"
label 'docker'
args "${DOCKER_ARGS}"
registryUrl 'https://docker.io/'
@@ -678,8 +428,8 @@
sh '''
cp js/centos-7/*rpm pkgs/centos7
cp js/centos-8/*rpm pkgs/centos8
- cd pkgs/centos7 && createrepo --database .
- cd ../centos8 && createrepo --database .
+ cd pkgs/centos7 && createrepo_c --database .
+ cd ../centos8 && createrepo_c --database .
'''
echo 'Building tree to upload...'
diff --git a/build-aux/README.Jenkins b/build-aux/README.md
similarity index 70%
rename from build-aux/README.Jenkins
rename to build-aux/README.md
index 014fbaa..72cef94 100644
--- a/build-aux/README.Jenkins
+++ b/build-aux/README.md
@@ -1,40 +1,6 @@
-# Building with a dedicated Jenkins CI instance
+# CouchDB and Jenkins CI
-## History
-
-Around April 2019, once Travis CI performance became so abysmal that the
-development team couldn't tolerate it any longer. We decided to move off
-of it entirely and onto our own dedicated Jenkins instance. Performance
-was bad because:
-
-* The Travis CI VMs are severely underpowered, leading to many
- unexpected test failures
-* The ASF wait queue for Travis was exceedingly long, with jobs
- sometimes taking 30-45 minutes just to _launch_
-* Our test suites (all 3 of Erlang, JavaScript and Elixir) had
- sporadic failures, requiring many re-launches of builds before
- we could move forward
-
-We also had other reasons to want to move to a dedicated server:
-
-* Credentials on the main Jenkins instance could be reused by *anyone*
- in the ASF
-* Credentials on the Travis CI setup should not contain credentials
- for things such as building binaries, to ensure the best possible
- security
-
-Other options we explored:
-
-* ASF BuildBot. This required a lot more work and couldn't guarantee
- availability of sufficient workers.
-* Circle CI, GitHub CI, etc.: Lack of availability of alternate binary
- platforms we've been asked to support (arm64v8, ppc64le) or alternate
- OSes (OSX, FreeBSD, Windows), though there is some support for either
- or both.
-
-## Cloudbees Core
-
-End of November 2019, ASF and CloudBees reached an agreement to allow
+In 2019, ASF and CloudBees reached an agreement to allow
the Foundation to use CloudBees Core to have a farm of managed Jenkins
masters. This allows the ASF to give larger projects their own dedicated
Jenkins master, which can be custom configured for the project. They can
@@ -53,15 +19,15 @@
project already had, this will provide the necessary compute resources
to meet the needs of the project for some time to come.
-# Jenkins configuration
+# Jenkins Configuration
All jobs on the new Jenkins master are contained in a CouchDB folder.
Credentials for the project are placed within this folder; this is a
unique capability of CloudBees Core at this time.
-## Pull Request job configuration
+## Pull Requests
-Blue Ocean link: https://ci-couchdb.apache.org/blue/organizations/jenkins/jenkins-cm1%2FPullRequests/activity/
+[Blue Ocean link](https://ci-couchdb.apache.org/blue/organizations/jenkins/jenkins-cm1%2FPullRequests/activity/)
To implement build-a-PR functionality in the same way that Travis
performs builds, Jenkins offers the Multibranch Pipeline job. Here's how
@@ -106,12 +72,15 @@
[2]: https://issues.apache.org/jira/browse/INFRA-17449 explains why "Discover pull requests from forks/Trust" has been set to "Nobody."
-## `master` and release branch configuration
+## Full Platform Builds on `main` and release branches
-Our old Jenkins job (formerly `/Jenkinsfile`) is now
-`build-aux/Jenkinsfile.full`. This builds CouchDB on `master`, all of
-our release branches (`2.*`, `3.*`, etc.) as well as `jenkins-*` for
-testing.
+[](https://ci-couchdb.apache.org/blue/organizations/jenkins/jenkins-cm1%2FFullPlatformMatrix/activity?branch=main)
+[](https://ci-couchdb.apache.org/blue/organizations/jenkins/jenkins-cm1%2FFullPlatformMatrix/activity?branch=3.x)
+
+Our original Jenkins job (formerly `/Jenkinsfile`) is now
+`build-aux/Jenkinsfile.full`. This builds CouchDB on `main`, all of
+our release branches (`2.*`, `3.*`, etc.) as well as any branch prefixed with
+`jenkins-` for testing on a wide variety of supported operating systems.
Settings are as follows:
@@ -145,5 +114,18 @@
* Max # of old items to keep: 10
* Everything else set as defaults.
+## Other Resources
+The [apache/couchdb-ci](https://github.com/apache/couchdb-ci) repo contains the
+dockerfiles that we use to generate the container images used for our
+container-based builds. These images are hosted on Docker Hub in the following
+repos:
+* [apache/couchdbci-debian](https://hub.docker.com/r/apache/couchdbci-debian)
+* [apache/couchdbci-ubuntu](https://hub.docker.com/r/apache/couchdbci-ubuntu)
+* [apache/couchdbci-centos](https://hub.docker.com/r/apache/couchdbci-centos)
+
+The [apache/couchdb-pkg](https://github.com/apache/couchdb-ci) repo contains
+a set of helper scripts to build binary packages for Debian / CentOS / Ubuntu
+from the contents of a release tarball. The packaging stage of our "Full
+Platform Builds" pipeline clones this repo to produces the package artifacts.
