Move apache dtests to running in docker
use dumb-init to get signal handling to work (https://engineeringblog.yelp.com/2016/01/dumb-init-an-init-for-docker.html)
Patch by marcuse; reviewed by Michael Shuler for CASSANDRA-14241
diff --git a/build-scripts/cassandra-dtest-pytest.sh b/build-scripts/cassandra-dtest-pytest.sh
index 993bcb3..7eae11e 100755
--- a/build-scripts/cassandra-dtest-pytest.sh
+++ b/build-scripts/cassandra-dtest-pytest.sh
@@ -52,13 +52,13 @@
rm -r upgrade_tests/ # TEMP: remove upgrade_tests - we have no dual JDK installation
set +e # disable immediate exit from this point
if [ "${DTEST_TARGET}" = "dtest" ]; then
- pytest --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
elif [ "${DTEST_TARGET}" = "dtest-novnode" ]; then
- pytest --log-level="INFO" --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ pytest -vv --log-level="INFO" --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
elif [ "${DTEST_TARGET}" = "dtest-offheap" ]; then
- pytest --log-level="INFO" --use-vnodes --num-tokens=32 --use-off-heap-memtables --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --use-off-heap-memtables --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
elif [ "${DTEST_TARGET}" = "dtest-large" ]; then
- pytest --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --force-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
+ pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --force-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt
else
echo "Unknown dtest target: ${DTEST_TARGET}"
exit 1
diff --git a/docker/jenkins/dtest.sh b/docker/jenkins/dtest.sh
index 87dd77f..0d8cdb9 100644
--- a/docker/jenkins/dtest.sh
+++ b/docker/jenkins/dtest.sh
@@ -3,7 +3,9 @@
export LANG=en_US.UTF-8
export PYTHONIOENCODING=utf-8
export PYTHONUNBUFFERED=true
+echo "dtest.sh: running: git clone --depth=1 --branch=$BRANCH https://github.com/$REPO/cassandra.git"
git clone --depth=1 --branch=$BRANCH https://github.com/$REPO/cassandra.git
cd cassandra
+echo git clone --branch=$DTEST_BRANCH $DTEST_REPO
git clone --branch=$DTEST_BRANCH $DTEST_REPO
-../cassandra-builds/build-scripts/cassandra-dtest-pytest.sh
+../cassandra-builds/build-scripts/cassandra-dtest-pytest.sh $1
diff --git a/docker/jenkins/jenkinscommand.sh b/docker/jenkins/jenkinscommand.sh
index f218ff9..5e4741d 100644
--- a/docker/jenkins/jenkinscommand.sh
+++ b/docker/jenkins/jenkinscommand.sh
@@ -1,13 +1,18 @@
#!/bin/sh
+DOCKER_IMAGE="kjellman/cassandra-test:0.4.4"
+BUILDSREPO=$5
+BUILDSBRANCH=$6
cat > env.list <<EOF
-REPO=$REPO
-BRANCH=$BRANCH
-DTEST_REPO=$DTEST_REPO
-DTEST_BRANCH=$DTEST_BRANCH
+REPO=$1
+BRANCH=$2
+DTEST_REPO=$3
+DTEST_BRANCH=$4
EOF
-ID=$(docker run --env-file env.list -dt kjellman/cassandra-test:0.4.3 bash -ilc "git clone --depth=1 --branch dock https://github.com/krummas/cassandra-builds.git; sh ./cassandra-builds/docker/jenkins/dtest.sh")
+echo "jenkinscommand.sh: running: git clone --depth=1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $7"
+ID=$(docker run --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --depth=1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $7")
# use docker attach instead of docker wait to get output
docker attach --no-stdin $ID
echo "$ID done, copying files"
docker cp $ID:/home/cassandra/cassandra/cassandra-dtest/nosetests.xml .
docker cp $ID:/home/cassandra/cassandra/test_stdout.txt .
+docker rm $ID
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 9ab039c..1d4d0da 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -344,7 +344,7 @@
scm(triggerInterval)
}
steps {
- shell("./cassandra-builds/build-scripts/cassandra-dtest-pytest.sh ${targetName}")
+ shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${targetName}")
}
}
}
@@ -423,7 +423,7 @@
}
/**
- * Parameterized Dev Branch dtest
+ * Parameterized Dev Branch dtest in docker
*/
job('Cassandra-devbranch-dtest') {
description(jobDescription)
@@ -446,7 +446,6 @@
stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout')
stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL')
stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout')
- stringParam('DTEST_SCRIPT', './cassandra-builds/build-scripts/cassandra-dtest-pytest.sh', 'A temporary means of specifying an alternate script to run the dtests.')
}
scm {
git {
@@ -462,62 +461,7 @@
steps {
buildDescription('', buildDescStr)
shell("git clean -xdff ; git clone -b ${buildsBranch} ${buildsRepo}")
- shell('git clone -b ${DTEST_BRANCH} ${DTEST_REPO}')
- shell('${DTEST_SCRIPT}')
- }
- publishers {
- archiveArtifacts('test_stdout.txt')
- archiveJunit('cassandra-dtest/nosetests.xml') {
- testDataPublishers {
- publishTestStabilityData()
- }
- }
- postBuildTask {
- task('.', 'echo "Finding job process orphans.."; if pgrep -af ${JOB_BASE_NAME}; then pkill -9 -f ${JOB_BASE_NAME}; fi')
- }
- }
-}
-
-/**
- * Parameterized Dev Branch dtest in docker
- */
-job('Cassandra-devbranch-dtest-docker') {
- description(jobDescription)
- concurrentBuild()
- jdk(jdkLabel)
- label(slaveLabel)
- logRotator {
- numToKeep(50)
- }
- wrappers {
- timeout {
- noActivity(2400)
- }
- }
- throttleConcurrentBuilds {
- categories(['Cassandra'])
- }
- parameters {
- stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from')
- stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout')
- stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL')
- stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout')
- }
- scm {
- git {
- remote {
- url('https://github.com/${REPO}/cassandra.git')
- }
- branch('${BRANCH}')
- extensions {
- cleanAfterCheckout()
- }
- }
- }
- steps {
- buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone -b dock https://github.com/krummas/cassandra-builds.git")
- shell('sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh')
+ shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch}")
}
publishers {
archiveArtifacts('test_stdout.txt')
