| //////////////////////////////////////////////////////////// |
| // |
| // Common Vars and Branch List |
| // |
| // Help on syntax see https://ci-cassandra.apache.org/plugin/job-dsl/api-viewer/index.html |
| // |
| // To update the variable via the Jenkins UI, use the EnvInject plugin |
| // example: https://github.com/apache/cassandra-builds/pull/19#issuecomment-610822772 |
| // |
| //////////////////////////////////////////////////////////// |
| |
| def jobDescription = '<img src="http://cassandra.apache.org/img/cassandra_logo.png" /><br/>Apache Cassandra DSL-generated job - DSL git repo: <a href="https://gitbox.apache.org/repos/asf?p=cassandra-builds.git">cassandra-builds</a>' |
| def jdkLabel = 'jdk_1.8_latest' |
| if(binding.hasVariable("CASSANDRA_JDK_LABEL")) { |
| jdkLabel = "${CASSANDRA_JDK_LABEL}" |
| } |
| def slaveLabel = 'cassandra' |
| if(binding.hasVariable("CASSANDRA_SLAVE_LABEL")) { |
| slaveLabel = "${CASSANDRA_SLAVE_LABEL}" |
| } |
| // The dtest-large target needs to run on >=32G slaves |
| def largeSlaveLabel = 'cassandra-large' |
| if(binding.hasVariable("CASSANDRA_LARGE_SLAVE_LABEL")) { |
| largeSlaveLabel = "${CASSANDRA_LARGE_SLAVE_LABEL}" |
| } |
| def mainRepo = "https://gitbox.apache.org/repos/asf/cassandra.git" |
| def githubRepo = "https://github.com/apache/cassandra" |
| if(binding.hasVariable("CASSANDRA_GIT_URL")) { |
| mainRepo = "${CASSANDRA_GIT_URL}" |
| // just presume custom repos are github, not critical if they are not |
| githubRepo = "${mainRepo}".minus(".git") |
| } |
| def buildsRepo = "https://gitbox.apache.org/repos/asf/cassandra-builds.git" |
| if(binding.hasVariable("CASSANDRA_BUILDS_GIT_URL")) { |
| buildsRepo = "${CASSANDRA_BUILDS_GIT_URL}" |
| } |
| def buildsBranch = "trunk" |
| if(binding.hasVariable("CASSANDRA_BUILDS_BRANCH")) { |
| buildsBranch = "${CASSANDRA_BUILDS_BRANCH}" |
| } |
| def dtestRepo = "https://gitbox.apache.org/repos/asf/cassandra-dtest.git" |
| if(binding.hasVariable("CASSANDRA_DTEST_GIT_URL")) { |
| dtestRepo = "${CASSANDRA_DTEST_GIT_URL}" |
| } |
| def buildDescStr = 'REF = ${GIT_BRANCH} <br /> COMMIT = ${GIT_COMMIT}' |
| // Cassandra active branches |
| def cassandraBranches = ['cassandra-2.2', 'cassandra-3.0', 'cassandra-3.11', 'trunk'] |
| if(binding.hasVariable("CASSANDRA_BRANCHES")) { |
| cassandraBranches = "${CASSANDRA_BRANCHES}".split(",") |
| } |
| // Ant test targets |
| def testTargets = ['test', 'test-burn', 'test-cdc', 'test-compression', 'stress-test', 'fqltool-test', 'long-test', 'jvm-dtest', 'jvm-dtest-upgrade'] |
| if(binding.hasVariable("CASSANDRA_ANT_TEST_TARGETS")) { |
| testTargets = "${CASSANDRA_ANT_TEST_TARGETS}".split(",") |
| } |
| |
| // Dtest test targets |
| def dtestTargets = ['dtest', 'dtest-novnode', 'dtest-offheap', 'dtest-large', 'dtest-upgrade'] |
| if(binding.hasVariable("CASSANDRA_DTEST_TEST_TARGETS")) { |
| dtestTargets = "${CASSANDRA_DTEST_TEST_TARGETS}".split(",") |
| } |
| def dtestDockerImage = 'nastra/cassandra-testing-ubuntu1910-java11' |
| if(binding.hasVariable("CASSANDRA_DOCKER_IMAGE")) { |
| dtestDockerImage = "${CASSANDRA_DOCKER_IMAGE}" |
| } |
| |
| // expected longest job runtime |
| def maxJobHours = '18' |
| if(binding.hasVariable("MAX_JOB_HOURS")) { |
| maxJobHours = "${MAX_JOB_HOURS}" |
| } |
| |
| // how many splits are dtest jobs matrixed into |
| def dtestSplits = 64 |
| def dtestLargeSplits = 8 |
| if(binding.hasVariable("DTEST_SPLITS")) { |
| dtestSplits = "${DTEST_SPLITS}" |
| } |
| if(binding.hasVariable("DTEST_LARGE_SPLITS")) { |
| dtestLargeSplits = "${DTEST_LARGE_SPLITS}" |
| } |
| |
| //////////////////////////////////////////////////////////// |
| // |
| // Job Templates |
| // - disabled by default |
| // - running jobs use templates for most configurations |
| // and set details like branch |
| // |
| //////////////////////////////////////////////////////////// |
| |
| /** |
| * Artifacts and eclipse-warnings template |
| */ |
| matrixJob('Cassandra-template-artifacts') { |
| disabled(true) |
| description(jobDescription) |
| concurrentBuild() |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(300) |
| } |
| timestamps() |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(1) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url(mainRepo) |
| } |
| branch('*/null') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; |
| echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ; |
| """) |
| } |
| publishers { |
| publishOverSsh { |
| server('Nightlies') { |
| transferSet { |
| sourceFiles("build/apache-cassandra-*.tar.gz, build/apache-cassandra-*.jar, build/apache-cassandra-*.pom, build/cassandra*.deb, build/cassandra*.rpm") |
| remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/") |
| } |
| } |
| failOnError(false) |
| } |
| extendedEmail { |
| recipientList('builds@cassandra.apache.org') |
| triggers { |
| failure { |
| sendTo { |
| recipientList() |
| developers() |
| requester() |
| culprits() |
| } |
| } |
| fixed { |
| sendTo { |
| recipientList() |
| developers() |
| requester() |
| culprits() |
| } |
| } |
| } |
| } |
| postBuildTask { |
| task('.', """ |
| echo "Cleaning project…"; git clean -xdff ; |
| echo "Pruning docker…" ; docker system prune -f --filter "until=${maxJobHours}h" ; |
| echo "Reporting disk usage…"; df -h ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| |
| /** |
| * Ant test template |
| */ |
| job('Cassandra-template-test') { |
| disabled(true) |
| description(jobDescription) |
| concurrentBuild() |
| jdk(jdkLabel) |
| label(slaveLabel) |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(1200) |
| } |
| timestamps() |
| } |
| throttleConcurrentBuilds { |
| categories(['Cassandra']) |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(3) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url(mainRepo) |
| } |
| branch('*/null') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; |
| echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ; |
| echo "\${JOB_BASE_NAME}: `git log -1 --pretty=format:'%h %an %ad %s'`" > \${JOB_BASE_NAME}.head ; |
| """) |
| } |
| publishers { |
| archiveArtifacts { |
| pattern('build/test/**/TEST-*.xml, **/*.head') |
| allowEmpty() |
| fingerprint() |
| } |
| archiveJunit('build/test/**/TEST-*.xml') { |
| testDataPublishers { |
| publishTestStabilityData() |
| } |
| } |
| publishOverSsh { |
| server('Nightlies') { |
| transferSet { |
| sourceFiles("build/test/logs/**") |
| remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/") |
| } |
| } |
| failOnError(false) |
| } |
| postBuildTask { |
| task('.', """ |
| echo "Finding job process orphans…"; if pgrep -af "\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi; |
| echo "Cleaning project…"; git clean -xdff ; |
| echo "Pruning docker…" ; docker system prune -f --filter "until=${maxJobHours}h" ; |
| echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs ../../* ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| |
| /** |
| * Dtest template |
| */ |
| matrixJob('Cassandra-template-dtest-matrix') { |
| disabled(true) |
| description(jobDescription) |
| concurrentBuild() |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(1200) |
| } |
| timestamps() |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(7) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url(mainRepo) |
| } |
| branch('*/null') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; |
| echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ; |
| echo "\${JOB_BASE_NAME}: `git log -1 --pretty=format:'%h %an %ad %s'`" > \${JOB_BASE_NAME}.head ; |
| """) |
| } |
| publishers { |
| publishOverSsh { |
| server('Nightlies') { |
| transferSet { |
| sourceFiles("**/nosetests.xml,**/test_stdout.txt.xz,**/ccm_logs.tar.xz") |
| remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/") |
| } |
| } |
| failOnError(false) |
| } |
| archiveArtifacts { |
| pattern('**/nosetests.xml, **/*.head') |
| allowEmpty() |
| fingerprint() |
| } |
| archiveJunit('nosetests.xml') { |
| testDataPublishers { |
| publishTestStabilityData() |
| } |
| } |
| postBuildTask { |
| // the pgrep needs to catch any other build/process that is using docker |
| task('.', """ |
| echo "Cleaning project…"; git clean -xdff ; |
| echo "Pruning docker…" ; if pgrep -af "cassandra-artifacts.sh|jenkinscommand.sh"; then docker system prune -f --filter 'until=${maxJobHours}h'; else docker system prune -f --volumes ; fi; |
| echo "Reporting disk usage…"; df -h ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| |
| /** |
| * cqlsh template |
| */ |
| matrixJob('Cassandra-template-cqlsh-tests') { |
| disabled(true) |
| description(jobDescription) |
| concurrentBuild() |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(1200) |
| } |
| timestamps() |
| } |
| throttleConcurrentBuilds { |
| categories(['Cassandra']) |
| } |
| axes { |
| text('cython', 'yes', 'no') |
| jdk(jdkLabel) |
| label('label', slaveLabel) |
| } |
| // this should prevent long path expansion from the axis definitions |
| childCustomWorkspace('.') |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(3) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url(mainRepo) |
| } |
| branch('*/null') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| echo "\${JOB_BASE_NAME}: `git log -1 --pretty=format:'%h %an %ad %s'`" > \${JOB_BASE_NAME}.head ; |
| """) |
| } |
| publishers { |
| archiveArtifacts { |
| pattern('**/cqlshlib.xml,**/nosetests.xml, **/*.head') |
| allowEmpty() |
| fingerprint() |
| } |
| archiveJunit('**/cqlshlib.xml,**/nosetests.xml') { |
| testDataPublishers { |
| publishTestStabilityData() |
| } |
| } |
| postBuildTask { |
| task('.', """ |
| echo "Finding job process orphans…"; if pgrep -af "\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi; |
| echo "Cleaning project…"; git clean -xdff ; |
| echo "Pruning docker…" ; docker system prune -f --filter "until=${maxJobHours}h" ; |
| echo "Reporting disk usage…"; df -h ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| |
| //////////////////////////////////////////////////////////// |
| // |
| // Branch Job Definitions |
| // - set to disabled(false) |
| // - running jobs use templates for most configurations |
| // and set details like branch |
| // |
| //////////////////////////////////////////////////////////// |
| |
| cassandraBranches.each { |
| def branchName = it |
| def jobNamePrefix = "Cassandra-${branchName}".replaceAll('cassandra-', '') |
| |
| /** |
| * Main branch artifacts and eclipse-warnings job |
| */ |
| matrixJob("${jobNamePrefix}-artifacts") { |
| disabled(false) |
| using('Cassandra-template-artifacts') |
| axes { |
| if (branchName == 'trunk') { |
| jdk('jdk_1.8_latest','jdk_11_latest') |
| } else { |
| jdk('jdk_1.8_latest') |
| } |
| label('label', slaveLabel) |
| } |
| configure { node -> |
| node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName) |
| } |
| steps { |
| shell('./cassandra-builds/build-scripts/cassandra-artifacts.sh') |
| } |
| } |
| |
| /** |
| * Main branch ant test target jobs |
| */ |
| testTargets.each { |
| def targetName = it |
| |
| // Skip tests that don't exist before cassandra-3.11 |
| if ((targetName == 'test-cdc' || targetName == 'stress-test') && ((branchName == 'cassandra-2.2') || (branchName == 'cassandra-3.0'))) { |
| println("Skipping ${targetName} on branch ${branchName}") |
| |
| // Skip tests that don't exist before cassandra-4.0 |
| } else if ((targetName == 'fqltool-test') && ((branchName == 'cassandra-2.2') || (branchName == 'cassandra-3.0') || (branchName == 'cassandra-3.11'))) { |
| println("Skipping ${targetName} on branch ${branchName}") |
| |
| } else { |
| job("${jobNamePrefix}-${targetName}") { |
| disabled(false) |
| using('Cassandra-template-test') |
| configure { node -> |
| node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName) |
| } |
| steps { |
| shell(""" |
| ./cassandra-builds/build-scripts/cassandra-test.sh ${targetName} ; |
| find build/test/logs -type f -name "*.log" | xargs xz -qq |
| """) |
| } |
| } |
| } |
| } |
| |
| /** |
| * Main branch dtest variation jobs |
| */ |
| dtestTargets.each { |
| def targetName = it |
| |
| // Skip dtest-offheap on cassandra-3.0 branch |
| if ((targetName == 'dtest-offheap') && (branchName == 'cassandra-3.0')) { |
| println("Skipping ${targetName} on branch ${branchName}") |
| } else { |
| matrixJob("${jobNamePrefix}-${targetName}") { |
| disabled(false) |
| using('Cassandra-template-dtest-matrix') |
| axes { |
| List<String> values = new ArrayList<String>() |
| if (targetName == 'dtest-large') { |
| splits = dtestLargeSplits |
| } else { |
| splits = dtestSplits |
| } |
| (1..splits).each { values << it.toString() } |
| text('split', values) |
| label('label', slaveLabel) |
| } |
| configure { node -> |
| node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName) |
| } |
| steps { |
| shell(""" |
| sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits} ; |
| xz test_stdout.txt |
| """) |
| } |
| } |
| } |
| } |
| |
| /** |
| * Main branch cqlsh jobs |
| */ |
| if (branchName == 'cassandra-2.2') { |
| println("Skipping ${jobNamePrefix}-cqlsh-tests, not supported on branch ${branchName}") |
| } else { |
| matrixJob("${jobNamePrefix}-cqlsh-tests") { |
| disabled(false) |
| using('Cassandra-template-cqlsh-tests') |
| configure { node -> |
| node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName) |
| } |
| steps { |
| shell("git clean -xdff") |
| shell('./pylib/cassandra-cqlsh-tests.sh $WORKSPACE') |
| } |
| } |
| } |
| |
| /** |
| * Branch Pipelines |
| */ |
| pipelineJob("${jobNamePrefix}") { |
| description(jobDescription) |
| compressBuildLog() |
| logRotator { |
| numToKeep(30) |
| artifactNumToKeep(10) |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(1) |
| } |
| } |
| definition { |
| cpsScm { |
| scm { |
| git { |
| remote { |
| url(mainRepo) |
| } |
| branch("${branchName}") |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| scriptPath('.jenkins/Jenkinsfile') |
| } |
| } |
| triggers { |
| scm('H/5 * * * *') |
| } |
| } |
| } |
| |
| //////////////////////////////////////////////////////////// |
| // |
| // Parameterized Dev Branch Job Definitions |
| // |
| //////////////////////////////////////////////////////////// |
| |
| /** |
| * Parameterized Artifacts |
| */ |
| matrixJob('Cassandra-devbranch-artifacts') { |
| description(jobDescription) |
| concurrentBuild() |
| axes { |
| jdk('jdk_1.8_latest','jdk_11_latest') |
| label('label', slaveLabel) |
| } |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(300) |
| } |
| timestamps() |
| } |
| parameters { |
| stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') |
| stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(1) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url('https://github.com/${REPO}/cassandra.git') |
| } |
| branch('${BRANCH}') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; |
| echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ; |
| """) |
| shell('./cassandra-builds/build-scripts/cassandra-artifacts.sh') |
| } |
| publishers { |
| publishOverSsh { |
| server('Nightlies') { |
| transferSet { |
| sourceFiles("build/apache-cassandra-*.tar.gz, build/apache-cassandra-*.jar, build/apache-cassandra-*.pom, build/cassandra*.deb, build/cassandra*.rpm") |
| remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/") |
| } |
| } |
| failOnError(false) |
| } |
| postBuildTask { |
| task('.', """ |
| echo "Cleaning project…"; git clean -xdff ; |
| echo "Pruning docker…" ; docker system prune -f --filter "until=${maxJobHours}h" ; |
| echo "Reporting disk usage…"; df -h ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type -f -atime +3 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| |
| /** |
| * Parameterized Dev Branch `ant test` |
| */ |
| testTargets.each { |
| def targetName = it |
| |
| job("Cassandra-devbranch-${targetName}") { |
| description(jobDescription) |
| concurrentBuild() |
| jdk(jdkLabel) |
| label(slaveLabel) |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(1200) |
| } |
| timestamps() |
| } |
| throttleConcurrentBuilds { |
| categories(['Cassandra']) |
| } |
| parameters { |
| stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') |
| stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(3) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url('https://github.com/${REPO}/cassandra.git') |
| } |
| branch('${BRANCH}') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; |
| echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ; |
| echo "Cassandra-devbranch-${targetName} cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head ; |
| """) |
| shell(""" |
| ./cassandra-builds/build-scripts/cassandra-test.sh ${targetName} ; |
| find build/test/logs -type f -name "*.log" | xargs xz -qq |
| """) |
| } |
| publishers { |
| publishOverSsh { |
| server('Nightlies') { |
| transferSet { |
| sourceFiles("build/test/logs/**") |
| remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/") |
| } |
| } |
| failOnError(false) |
| } |
| archiveArtifacts { |
| pattern('build/test/**/TEST-*.xml, **/*.head') |
| allowEmpty() |
| fingerprint() |
| } |
| archiveJunit('build/test/**/TEST-*.xml') { |
| allowEmptyResults() |
| } |
| postBuildTask { |
| task('.', """ |
| echo "Finding job process orphans…"; if pgrep -af "\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi; |
| echo "Cleaning project…"; git clean -xdff ; |
| echo "Pruning docker…" ; docker system prune -f --filter "until=${maxJobHours}h" ; |
| echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs ../../* ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| } |
| |
| /** |
| * Parameterized Dev Branch dtest in docker. |
| * |
| * Only the vanilla dtest target is used in the Cassandra-devbranch pipeline, |
| * but they are all added here for developers needing to pre-commit test them specifically. |
| */ |
| dtestTargets.each { |
| def targetName = it |
| |
| matrixJob("Cassandra-devbranch-${targetName}") { |
| description(jobDescription) |
| concurrentBuild() |
| compressBuildLog() |
| jdk(jdkLabel) |
| if (targetName == 'dtest-large') { |
| label(largeSlaveLabel) |
| } else { |
| label(slaveLabel) |
| } |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(2400) |
| } |
| timestamps() |
| } |
| parameters { |
| stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') |
| stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') |
| stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') |
| stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout') |
| stringParam('DOCKER_IMAGE', "${dtestDockerImage}", 'Docker image for running dtests') |
| } |
| axes { |
| List<String> values = new ArrayList<String>() |
| if (targetName == 'dtest-large') { |
| splits = dtestLargeSplits |
| } else { |
| splits = dtestSplits |
| } |
| (1..splits).each { values << it.toString() } |
| text('split', values) |
| label('label', slaveLabel) |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(6) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url('https://github.com/${REPO}/cassandra.git') |
| } |
| branch('${BRANCH}') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; |
| echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ; |
| echo "Cassandra-devbranch-${targetName} cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head ; |
| """) |
| shell(""" |
| sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${splits} ; |
| xz test_stdout.txt |
| """) |
| } |
| publishers { |
| publishOverSsh { |
| server('Nightlies') { |
| transferSet { |
| sourceFiles("**/test_stdout.txt.xz,**/ccm_logs.tar.xz") |
| remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/") |
| } |
| } |
| failOnError(false) |
| } |
| archiveArtifacts { |
| pattern('**/nosetests.xml, **/*.head') |
| allowEmpty() |
| fingerprint() |
| } |
| archiveJunit('nosetests.xml') |
| postBuildTask { |
| // the pgrep needs to catch any other build/process that is using docker |
| task('.', """ |
| echo "Cleaning project…" ; git clean -xdff ; |
| echo "Pruning docker…" ; if pgrep -af "cassandra-artifacts.sh|jenkinscommand.sh"; then docker system prune -f --filter "until=${maxJobHours}h"; else docker system prune -f --volumes ; fi; |
| echo "Reporting disk usage…"; df -h ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| } |
| |
| |
| /** |
| * Parameterized Dev Branch cqlsh-tests |
| */ |
| matrixJob('Cassandra-devbranch-cqlsh-tests') { |
| description(jobDescription) |
| concurrentBuild() |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| timeout { |
| noActivity(1200) |
| } |
| timestamps() |
| } |
| throttleConcurrentBuilds { |
| categories(['Cassandra']) |
| } |
| parameters { |
| stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') |
| stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') |
| stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') |
| stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout') |
| } |
| axes { |
| text('cython', 'yes', 'no') |
| jdk(jdkLabel) |
| label('label', slaveLabel) |
| } |
| // this should prevent long path expansion from the axis definitions |
| childCustomWorkspace('.') |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(3) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url('https://github.com/${REPO}/cassandra.git') |
| } |
| branch('${BRANCH}') |
| extensions { |
| cleanAfterCheckout() |
| } |
| } |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| shell(""" |
| git clean -xdff ; |
| echo "Cassandra-devbranch-cqlsh-tests: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-cqlsh-tests.head ; |
| """) |
| shell('./pylib/cassandra-cqlsh-tests.sh $WORKSPACE') |
| } |
| publishers { |
| archiveArtifacts { |
| pattern('**/cqlshlib.xml,**/nosetests.xml, **/*.head') |
| allowEmpty() |
| fingerprint() |
| } |
| archiveJunit('**/cqlshlib.xml,**/nosetests.xml') |
| postBuildTask { |
| task('.', """ |
| echo "Finding job process orphans…"; if pgrep -af "\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi; |
| echo "Cleaning project…"; git clean -xdff ; |
| echo "Pruning docker…" ; docker system prune -f --filter "until=${maxJobHours}h" ; |
| echo "Reporting disk usage…"; df -h ; |
| echo "Cleaning tmp…"; |
| find . -type d -name tmp -delete 2>/dev/null ; |
| find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null |
| """) |
| } |
| } |
| } |
| |
| |
| /** |
| * Parameterized Dev Branch Pipeline |
| */ |
| pipelineJob('Cassandra-devbranch') { |
| description(jobDescription) |
| compressBuildLog() |
| logRotator { |
| numToKeep(30) |
| artifactNumToKeep(10) |
| } |
| parameters { |
| stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') |
| stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') |
| stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') |
| stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout') |
| stringParam('DOCKER_IMAGE', "${dtestDockerImage}", 'Docker image for running dtests') |
| } |
| properties { |
| githubProjectUrl(githubRepo) |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(1) |
| } |
| } |
| definition { |
| cps { |
| // Cassandra-devbranch still needs custom Jenkinsfile because of the parameters passed into the build jobs. |
| script(readFileFromWorkspace('Cassandra-Job-DSL', 'jenkins-dsl/cassandra_pipeline.groovy')) |
| } |
| } |
| } |
| |
| //////////////////////////////////////////////////////////// |
| // |
| // Jobs for Other Cassandra Projects |
| // |
| //////////////////////////////////////////////////////////// |
| |
| job('cassandra-website') { |
| description(jobDescription) |
| label('git-websites') |
| compressBuildLog() |
| logRotator { |
| numToKeep(10) |
| artifactNumToKeep(10) |
| } |
| wrappers { |
| preBuildCleanup() |
| timeout { |
| noActivity(300) |
| } |
| timestamps() |
| } |
| properties { |
| githubProjectUrl('https://github.com/apache/cassandra-website/') |
| priorityJobProperty { |
| useJobPriority(true) |
| priority(1) |
| } |
| } |
| scm { |
| git { |
| remote { |
| url('https://gitbox.apache.org/repos/asf/cassandra-website.git') |
| credentials('9b041bd0-aea9-4498-a576-9eeb771411dd') // "jenkins" |
| } |
| branch('*/trunk') |
| extensions { |
| wipeOutWorkspace() |
| cleanBeforeCheckout() |
| cleanAfterCheckout() |
| } |
| } |
| } |
| triggers { |
| scm('H/5 * * * *') |
| } |
| steps { |
| buildDescription('', buildDescStr) |
| // the chmod below is a hack for INFRA-20814 |
| // for debugging it can be useful to add a `git show --stat HEAD` before the push |
| shell(""" |
| git checkout asf-staging ; |
| git reset --hard origin/trunk ; |
| docker-compose build --build-arg UID=`id -u` --build-arg GID=`id -g` cassandra-website ; |
| chmod -R 777 src content ; |
| docker-compose run cassandra-website ; |
| git add content/ src/doc/ ; |
| git commit -a -m "generate docs for `git rev-parse --short HEAD`" ; |
| git push -f origin asf-staging ; |
| """) |
| } |
| } |
