In Jenkins parallelise (matrix) dtest-upgrade runs
patch by Mick Semb Wever; reviewed by Eduard Tudenhöfner
diff --git a/build-scripts/cassandra-dtest-pytest.sh b/build-scripts/cassandra-dtest-pytest.sh
index cad2e62..e099ba4 100755
--- a/build-scripts/cassandra-dtest-pytest.sh
+++ b/build-scripts/cassandra-dtest-pytest.sh
@@ -8,6 +8,8 @@
# Pass in target to run, default to base dtest
DTEST_TARGET="${1:-dtest}"
+# Optional: pass in chunk to test, formatted as "K/N" for the Kth chunk of N chunks
+DTEST_SPLIT_CHUNK="$2"
export PYTHONIOENCODING="utf-8"
export PYTHONUNBUFFERED=true
@@ -37,6 +39,7 @@
break
fi
done
+
# Exit, if we didn't build successfully
if [ "${RETURN}" -ne "0" ]; then
echo "Build failed with exit code: ${RETURN}"
@@ -65,21 +68,31 @@
mkdir -p ${TMPDIR}
set +e # disable immediate exit from this point
if [ "${DTEST_TARGET}" = "dtest" ]; then
- pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml --junit-prefix=${DTEST_TARGET} -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --skip-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-novnode" ]; then
- pytest -vv --log-level="INFO" --junit-xml=nosetests.xml --junit-prefix=${DTEST_TARGET} -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ DTEST_ARGS="--skip-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-offheap" ]; then
- pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --use-off-heap-memtables --junit-xml=nosetests.xml --junit-prefix=${DTEST_TARGET} -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --use-off-heap-memtables --skip-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-large" ]; then
- pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml --junit-prefix=${DTEST_TARGET} -s --cassandra-dir=$CASSANDRA_DIR --only-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --only-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-upgrade" ]; then
+ DTEST_ARGS="--execute-upgrade-tests-only "
export RUN_STATIC_UPGRADE_MATRIX=true
- pytest -vv --log-level="INFO" --execute-upgrade-tests-only --junit-xml=nosetests.xml --junit-prefix=${DTEST_TARGET} -s --cassandra-dir=$CASSANDRA_DIR 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
else
echo "Unknown dtest target: ${DTEST_TARGET}"
exit 1
fi
+SPLIT_TESTS=""
+if [ "x${DTEST_SPLIT_CHUNK}" != "x" ] ; then
+ ./run_dtests.py --cassandra-dir=$CASSANDRA_DIR ${DTEST_ARGS} --dtest-print-tests-only --dtest-print-tests-output=${WORKSPACE}/test_list.txt 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ SPLIT_TESTS=$(split -n l/${DTEST_SPLIT_CHUNK} ${WORKSPACE}/test_list.txt)
+fi
+
+PYTEST_OPTS="-vv --log-level="INFO" --junit-xml=nosetests.xml --junit-prefix=${DTEST_TARGET} -s"
+
+pytest ${PYTEST_OPTS} --cassandra-dir=$CASSANDRA_DIR ${DTEST_ARGS} ${SPLIT_TESTS} 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+
# tar up any ccm logs for easy retrieval
tar -cJf ccm_logs.tar.xz ${TMPDIR}/*/test/*/logs/*
diff --git a/docker/jenkins/dtest.sh b/docker/jenkins/dtest.sh
index 47ec642..417a36b 100644
--- a/docker/jenkins/dtest.sh
+++ b/docker/jenkins/dtest.sh
@@ -9,4 +9,4 @@
cd cassandra
echo git clone --depth 1 --single-branch --branch=$DTEST_BRANCH $DTEST_REPO
git clone --depth 1 --single-branch --branch=$DTEST_BRANCH $DTEST_REPO
-../cassandra-builds/build-scripts/cassandra-dtest-pytest.sh $1
+../cassandra-builds/build-scripts/cassandra-dtest-pytest.sh "$@"
diff --git a/docker/jenkins/jenkinscommand.sh b/docker/jenkins/jenkinscommand.sh
index bc2b835..7ce3c65 100644
--- a/docker/jenkins/jenkinscommand.sh
+++ b/docker/jenkins/jenkinscommand.sh
@@ -7,6 +7,7 @@
BUILDSBRANCH=$6
DOCKER_IMAGE=$7
TARGET=$8
+SPLIT_CHUNK=$9
cat > env.list <<EOF
REPO=$1
BRANCH=$2
@@ -14,8 +15,8 @@
DTEST_BRANCH=$4
EOF
-echo "jenkinscommand.sh: running: git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $TARGET"
-ID=$(docker run --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $TARGET")
+echo "jenkinscommand.sh: running: git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $TARGET $SPLIT_CHUNK"
+ID=$(docker run --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $TARGET $SPLIT_CHUNK")
# use docker attach instead of docker wait to get output
docker attach --no-stdin $ID
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 39bc2b1..7976d73 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -275,6 +275,64 @@
}
}
+// wip – parallelise dtests, starting with just upgrade dtests
+matrixJob('Cassandra-template-dtest-matrix') {
+ disabled(true)
+ description(jobDescription)
+ concurrentBuild()
+ compressBuildLog()
+ logRotator {
+ numToKeep(10)
+ artifactNumToKeep(10)
+ }
+ wrappers {
+ timeout {
+ noActivity(1200)
+ }
+ timestamps()
+ }
+ properties {
+ githubProjectUrl(githubRepo)
+ }
+ scm {
+ git {
+ remote {
+ url(mainRepo)
+ }
+ branch('*/null')
+ extensions {
+ cleanAfterCheckout()
+ }
+ }
+ }
+ steps {
+ buildDescription('', buildDescStr)
+ shell("git clean -xdff ; git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; git clone --depth 1 --single-branch ${dtestRepo}")
+ }
+ publishers {
+ archiveArtifacts {
+ pattern('**/test_stdout.txt,**/nosetests.xml,**/ccm_logs.tar.xz')
+ allowEmpty()
+ fingerprint()
+ }
+ archiveJunit('nosetests.xml') {
+ testDataPublishers {
+ publishTestStabilityData()
+ }
+ }
+ postBuildTask {
+ task('.', '''
+ echo "Cleaning project…"; git clean -xdff ;
+ echo "Pruning docker…" ; if pgrep -af jenkinscommand.sh; then docker system prune -f --filter 'until=48h'; else docker system prune -f --volumes ; fi;
+ echo "Reporting disk usage…"; df -h ; du -hs ../* ;
+ echo "Cleaning tmp…";
+ find . -type d -name tmp -delete 2>/dev/null ;
+ find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null
+ ''')
+ }
+ }
+}
+
/**
* cqlsh template
*/
@@ -418,6 +476,25 @@
// Skip dtest-offheap on cassandra-3.0 branch
if ((targetName == 'dtest-offheap') && (branchName == 'cassandra-3.0')) {
println("Skipping ${targetName} on branch ${branchName}")
+ } else if (targetName == 'dtest-upgrade') {
+ // wip – parallelise dtests, starting with just upgrade dtests
+ matrixJob("${jobNamePrefix}-${targetName}") {
+ disabled(false)
+ using('Cassandra-template-dtest-matrix')
+ axes {
+ splits = 64
+ List<String> values = new ArrayList<String>()
+ (1..splits).each { values << it.toString() }
+ text('split', values)
+ label('label', slaveLabel)
+ }
+ configure { node ->
+ node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName)
+ }
+ steps {
+ shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits}")
+ }
+ }
} else {
job("${jobNamePrefix}-${targetName}") {
disabled(false)
