| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| version: 2 |
| jobs: |
| j8_jvm_upgrade_dtests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine distributed Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_fqltool_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_cqlshlib_cython_tests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run cqlshlib Unit Tests |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| export cython="yes" |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra/ |
| ./pylib/cassandra-cqlsh-tests.sh $(pwd) |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/pylib |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlsh_dtests_py311_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_with_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt |
| |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.11 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_dtests_vnode_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if true; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_dtests_large_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_large_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_large_with_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_large_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_large_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_system_keyspace_directory: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist-system-keyspace-directory) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_utests_stress: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run Unit Tests (stress-test) |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| ant stress-test -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_stress_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_cqlsh_dtests_py311: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_without_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt |
| |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.11 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_jvm_dtests_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine distributed Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_utests_compression_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_unit_tests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlsh_dtests_py3: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_without_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.6 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlsh_dtests_py38: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_without_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt |
| |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.8 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_utests_compression_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-compression\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_repeated_ant_test: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run repeated JUnit test |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then |
| echo "Repeated utest class name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then |
| echo "Repeated utest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then |
| echo "Repeated utest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times" |
| |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| |
| target=${REPEATED_ANT_TEST_TARGET} |
| class_path=${REPEATED_ANT_TEST_CLASS} |
| class_name="${class_path##*.}" |
| |
| # Prepare the -Dtest.name argument. |
| # It can be the fully qualified class name or the short class name, depending on the target. |
| if [[ $target == "test" || \ |
| $target == "test-cdc" || \ |
| $target == "test-compression" || \ |
| $target == "test-system-keyspace-directory" || \ |
| $target == "fqltool-test" || \ |
| $target == "long-test" || \ |
| $target == "stress-test" || \ |
| $target == "test-simulator-dtest" ]]; then |
| name="-Dtest.name=$class_name" |
| else |
| name="-Dtest.name=$class_path" |
| fi |
| |
| # Prepare the -Dtest.methods argument, which is optional |
| if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then |
| methods="" |
| else |
| methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}" |
| fi |
| |
| # Prepare the JVM dtests vnodes argument, which is optional |
| vnodes_args="" |
| if ${REPEATED_ANT_TEST_VNODES}; then |
| vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'" |
| fi |
| |
| # Run the test target as many times as requested collecting the exit code, |
| # stopping the iteration only if stop_on_failure is set. |
| exit_code="$?" |
| for i in $(seq -w 1 $count); do |
| |
| echo "Running test iteration $i of $count" |
| |
| # run the test |
| status="passes" |
| if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then |
| status="fails" |
| exit_code=1 |
| fi |
| |
| # move the stdout output file |
| dest=/tmp/results/repeated_utest/stdout/${status}/${i} |
| mkdir -p $dest |
| mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt |
| |
| # move the XML output files |
| source=build/test/output |
| dest=/tmp/results/repeated_utest/output/${status}/${i} |
| mkdir -p $dest |
| if [[ -d $source && -n "$(ls $source)" ]]; then |
| mv $source/* $dest/ |
| fi |
| |
| # move the log files |
| source=build/test/logs |
| dest=/tmp/results/repeated_utest/logs/${status}/${i} |
| mkdir -p $dest |
| if [[ -d $source && -n "$(ls $source)" ]]; then |
| mv $source/* $dest/ |
| fi |
| |
| # maybe stop iterations on test failure |
| if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then |
| break |
| fi |
| done |
| |
| (exit ${exit_code}) |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results/repeated_utest/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utest/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utest/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utest/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_dtests_large_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_large_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_with_vnodes_raw /tmp/all_dtest_tests_j11_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_with_vnodes_raw > /tmp/all_dtest_tests_j11_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_with_vnodes > /tmp/split_dtest_tests_j11_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_large_with_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_large_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_large_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_dtests_large_vnode_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_LARGE_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if true; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlsh_dtests_py311: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_without_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt |
| |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.11 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlsh_dtests_py38_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j11_dtests_offheap) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" |
| cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt |
| |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.8 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_dtests_large: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_large_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_large_without_vnodes_raw /tmp/all_dtest_tests_j11_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_large_without_vnodes_raw > /tmp/all_dtest_tests_j11_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_large_without_vnodes > /tmp/split_dtest_tests_j11_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_large_without_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_large_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_large_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_utests_system_keyspace_directory_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlsh_dtests_py3_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_with_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.6 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlsh_dtests_py3: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_without_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.6 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlshlib_cython_tests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run cqlshlib Unit Tests |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| export cython="yes" |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra/ |
| ./pylib/cassandra-cqlsh-tests.sh $(pwd) |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/pylib |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_cdc: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist-cdc) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_utests_fqltool: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run Unit Tests (fqltool-test) |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| ant fqltool-test -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_utests_system_keyspace_directory: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist-system-keyspace-directory) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_dtests_offheap_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if true; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_dtests_offheap_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if true; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_dtests_large_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_LARGE_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if false; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_jvm_dtests_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine distributed Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_simulator_dtests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Simulator Tests |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| ant test-simulator-dtest -Dno-build-test=true |
| no_output_timeout: 30m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_utests_compression: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist-compression) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_long: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run Unit Tests (long-test) |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| ant long-test -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_unit_tests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_dtests_large: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_large_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_large_without_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_large_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_large_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_stress: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run Unit Tests (stress-test) |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| ant stress-test -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlsh_dtests_py38_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j8_dtests_offheap) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" |
| cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt |
| |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.8 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_upgrade_dtests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if false; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if true; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_cdc_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_dtests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if false; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_utests_fqltool_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=fqltool-test\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_jvm_dtests_vnode_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_compression: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist-compression) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlsh_dtests_py38: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_without_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt |
| |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.8 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlsh_dtests_py3_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j11_dtests_offheap) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" |
| cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.6 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_simulator_dtests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-simulator-dtest\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlsh_dtests_py311_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j11_dtests_offheap) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" |
| cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt |
| |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.11 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_utests_system_keyspace_directory_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-system-keyspace-directory\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_dtests_large_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_LARGE_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if false; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_cqlsh_dtests_py3_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j8_dtests_offheap) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" |
| cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.6 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_utests_stress_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=stress-test-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_dtests_vnode_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if true; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlsh_dtests_py3_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_with_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.6 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_upgrade_dtests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_upgradetests_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_upgradetests_without_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_upgradetests_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_upgradetests_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_dtests_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j11_dtests_offheap) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_jvm_upgrade_dtests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlsh_dtests_py38_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_with_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt |
| |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.8 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_jvm_dtests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_cqlsh_dtests_py311_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_with_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt |
| |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.11 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_utests_long_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_dtests_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j8_dtests_offheap) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_unit_tests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_jvm_dtests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine distributed Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_jvm_dtests_vnode_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j11_build: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Clone Cassandra Repository (via git) |
| command: | |
| git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra |
| - run: |
| name: Build Cassandra |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| cd ~/cassandra |
| # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. |
| for x in $(seq 1 3); do |
| ${ANT_HOME}/bin/ant clean realclean jar |
| RETURN="$?" |
| if [ "${RETURN}" -eq "0" ]; then |
| break |
| fi |
| done |
| # Exit, if we didn't build successfully |
| if [ "${RETURN}" -ne "0" ]; then |
| echo "Build failed with exit code: ${RETURN}" |
| exit ${RETURN} |
| fi |
| no_output_timeout: 15m |
| - run: |
| name: Run eclipse-warnings |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| cd ~/cassandra |
| ant eclipse-warnings |
| - persist_to_workspace: |
| root: /home/cassandra |
| paths: |
| - cassandra |
| - .m2 |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_dtests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_without_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_dtests_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_with_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_cqlshlib_tests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run cqlshlib Unit Tests |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra/ |
| ./pylib/cassandra-cqlsh-tests.sh $(pwd) |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/pylib |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_jvm_dtests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_dtests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_without_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_without_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_without_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_without_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_repeated_ant_test: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run repeated JUnit test |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then |
| echo "Repeated utest class name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then |
| echo "Repeated utest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then |
| echo "Repeated utest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times" |
| |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| |
| target=${REPEATED_ANT_TEST_TARGET} |
| class_path=${REPEATED_ANT_TEST_CLASS} |
| class_name="${class_path##*.}" |
| |
| # Prepare the -Dtest.name argument. |
| # It can be the fully qualified class name or the short class name, depending on the target. |
| if [[ $target == "test" || \ |
| $target == "test-cdc" || \ |
| $target == "test-compression" || \ |
| $target == "test-system-keyspace-directory" || \ |
| $target == "fqltool-test" || \ |
| $target == "long-test" || \ |
| $target == "stress-test" || \ |
| $target == "test-simulator-dtest" ]]; then |
| name="-Dtest.name=$class_name" |
| else |
| name="-Dtest.name=$class_path" |
| fi |
| |
| # Prepare the -Dtest.methods argument, which is optional |
| if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then |
| methods="" |
| else |
| methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}" |
| fi |
| |
| # Prepare the JVM dtests vnodes argument, which is optional |
| vnodes_args="" |
| if ${REPEATED_ANT_TEST_VNODES}; then |
| vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'" |
| fi |
| |
| # Run the test target as many times as requested collecting the exit code, |
| # stopping the iteration only if stop_on_failure is set. |
| exit_code="$?" |
| for i in $(seq -w 1 $count); do |
| |
| echo "Running test iteration $i of $count" |
| |
| # run the test |
| status="passes" |
| if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then |
| status="fails" |
| exit_code=1 |
| fi |
| |
| # move the stdout output file |
| dest=/tmp/results/repeated_utest/stdout/${status}/${i} |
| mkdir -p $dest |
| mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt |
| |
| # move the XML output files |
| source=build/test/output |
| dest=/tmp/results/repeated_utest/output/${status}/${i} |
| mkdir -p $dest |
| if [[ -d $source && -n "$(ls $source)" ]]; then |
| mv $source/* $dest/ |
| fi |
| |
| # move the log files |
| source=build/test/logs |
| dest=/tmp/results/repeated_utest/logs/${status}/${i} |
| mkdir -p $dest |
| if [[ -d $source && -n "$(ls $source)" ]]; then |
| mv $source/* $dest/ |
| fi |
| |
| # maybe stop iterations on test failure |
| if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then |
| break |
| fi |
| done |
| |
| (exit ${exit_code}) |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results/repeated_utest/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utest/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utest/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utest/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_utests_long: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run Unit Tests (long-test) |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| ant long-test -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_utests_cdc: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine unit Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist-cdc) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_cqlsh_dtests_py311_offheap: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_dtests_offheap) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" |
| - run: |
| name: Run dtests (j8_dtests_offheap) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" |
| cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt |
| |
| source ~/env3.11/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.11 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_dtests_offheap |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_dtests_offheap_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_dtests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if false; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_jvm_dtests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Determine distributed Tests to Run |
| command: | |
| # reminder: this code (along with all the steps) is independently executed on every circle container |
| # so the goal here is to get the circleci script to return the tests *this* container will run |
| # which we do via the `circleci` cli tool. |
| |
| rm -fr ~/cassandra-dtest/upgrade_tests |
| echo "***java tests***" |
| |
| # get all of our unit test filenames |
| set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt |
| |
| # split up the unit tests into groups based on the number of containers we have |
| set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
| set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" |
| cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt |
| no_output_timeout: 15m |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Run Unit Tests (testclasslist) |
| command: | |
| set -x |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) |
| if [ -z "$test_timeout" ]; then |
| test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') |
| fi |
| ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_build: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Clone Cassandra Repository (via git) |
| command: | |
| git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra |
| - run: |
| name: Build Cassandra |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| cd ~/cassandra |
| # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. |
| for x in $(seq 1 3); do |
| ${ANT_HOME}/bin/ant clean realclean jar |
| RETURN="$?" |
| if [ "${RETURN}" -eq "0" ]; then |
| break |
| fi |
| done |
| # Exit, if we didn't build successfully |
| if [ "${RETURN}" -ne "0" ]; then |
| echo "Build failed with exit code: ${RETURN}" |
| exit ${RETURN} |
| fi |
| no_output_timeout: 15m |
| - run: |
| name: Run eclipse-warnings |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| cd ~/cassandra |
| ant eclipse-warnings |
| - persist_to_workspace: |
| root: /home/cassandra |
| paths: |
| - cassandra |
| - .m2 |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_cqlsh_dtests_py38_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j8_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j8_with_vnodes) |
| no_output_timeout: 15m |
| command: | |
| echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" |
| cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt |
| |
| source ~/env3.8/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then |
| export CQLSH_PYTHON=/usr/bin/python3.8 |
| fi |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` |
| if [ ! -z "$SPLIT_TESTS" ]; then |
| set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt |
| else |
| echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" |
| (exit 1) |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j8_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j8_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_unit_tests_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_utests_fqltool: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run Unit Tests (fqltool-test) |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra |
| if [ -d ~/dtest_jars ]; then |
| cp ~/dtest_jars/dtest* /tmp/cassandra/build/ |
| fi |
| ant fqltool-test -Dno-build-test=true |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/build/test/output/ |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/cassandra/build/test/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_cqlshlib_tests: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Run cqlshlib Unit Tests |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| time mv ~/cassandra /tmp |
| cd /tmp/cassandra/ |
| ./pylib/cassandra-cqlsh-tests.sh $(pwd) |
| no_output_timeout: 15m |
| - store_test_results: |
| path: /tmp/cassandra/pylib |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_dtests_vnode: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Determine Tests to Run (j11_with_vnodes) |
| no_output_timeout: 5m |
| command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" |
| - run: |
| name: Run dtests (j11_with_vnodes) |
| no_output_timeout: 15m |
| command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest_j11_with_vnodes |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_j11_with_vnodes_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_utests_cdc_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-cdc\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-cdc $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j8_dtests_large_vnode_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Clone Cassandra dtest Repository (via git) |
| command: | |
| git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest |
| - run: |
| name: Configure virtualenv and python Dependencies |
| command: | |
| # note, this should be super quick as all dependencies should be pre-installed in the docker image |
| # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated |
| # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and |
| # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt |
| pip3 uninstall -y cqlsh |
| pip3 freeze |
| - run: |
| name: Run repeated Python DTests |
| no_output_timeout: 15m |
| command: | |
| if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then |
| echo "Repeated dtest name hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then |
| echo "Repeated dtest count hasn't been defined, exiting without running any test" |
| elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then |
| echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" |
| else |
| |
| # Calculate the number of test iterations to be run by the current parallel runner. |
| # Since we are running the same test multiple times there is no need to use `circleci tests split`. |
| count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) |
| if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then |
| count=$((count+1)) |
| fi |
| |
| if (($count <= 0)); then |
| echo "No tests to run in this runner" |
| else |
| echo "Running ${REPEATED_LARGE_DTESTS} $count times" |
| |
| source ~/env3.6/bin/activate |
| export PATH=$JAVA_HOME/bin:$PATH |
| |
| java -version |
| cd ~/cassandra-dtest |
| mkdir -p /tmp/dtest |
| |
| echo "env: $(env)" |
| echo "** done env" |
| mkdir -p /tmp/results/dtests |
| |
| tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g") |
| |
| stop_on_failure_arg="" |
| if ${REPEATED_TESTS_STOP_ON_FAILURE}; then |
| stop_on_failure_arg="-x" |
| fi |
| |
| vnodes_args="" |
| if true; then |
| vnodes_args="--use-vnodes --num-tokens=16" |
| fi |
| |
| upgrade_arg="" |
| if false; then |
| upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" |
| fi |
| |
| # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee |
| set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt |
| fi |
| fi |
| - store_test_results: |
| path: /tmp/results |
| - store_artifacts: |
| path: /tmp/dtest |
| destination: dtest |
| - store_artifacts: |
| path: ~/cassandra-dtest/logs |
| destination: dtest_logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| j11_utests_long_repeat: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 4 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Log Environment Information |
| command: | |
| echo '*** id ***' |
| id |
| echo '*** cat /proc/cpuinfo ***' |
| cat /proc/cpuinfo |
| echo '*** free -m ***' |
| free -m |
| echo '*** df -m ***' |
| df -m |
| echo '*** ifconfig -a ***' |
| ifconfig -a |
| echo '*** uname -a ***' |
| uname -a |
| echo '*** mount ***' |
| mount |
| echo '*** env ***' |
| env |
| echo '*** java ***' |
| which java |
| java -version |
| - run: |
| name: Repeatedly run new or modifed JUnit tests |
| no_output_timeout: 15m |
| command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=long-testsome\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" |
| - store_test_results: |
| path: /tmp/results/repeated_utests/output |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/stdout |
| destination: stdout |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/output |
| destination: junitxml |
| - store_artifacts: |
| path: /tmp/results/repeated_utests/logs |
| destination: logs |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - CASSANDRA_USE_JDK11: true |
| j8_dtest_jars_build: |
| docker: |
| - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest |
| resource_class: medium |
| working_directory: ~/ |
| shell: /bin/bash -eo pipefail -l |
| parallelism: 1 |
| steps: |
| - attach_workspace: |
| at: /home/cassandra |
| - run: |
| name: Build Cassandra DTest jars |
| command: | |
| export PATH=$JAVA_HOME/bin:$PATH |
| cd ~/cassandra |
| mkdir ~/dtest_jars |
| git remote add apache https://github.com/apache/cassandra.git |
| for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do |
| # check out the correct cassandra version: |
| git remote set-branches --add apache '$branch' |
| git fetch --depth 1 apache $branch |
| git checkout $branch |
| git clean -fd |
| # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. |
| for x in $(seq 1 3); do |
| ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar |
| RETURN="$?" |
| if [ "${RETURN}" -eq "0" ]; then |
| cp build/dtest*.jar ~/dtest_jars |
| break |
| fi |
| done |
| # Exit, if we didn't build successfully |
| if [ "${RETURN}" -ne "0" ]; then |
| echo "Build failed with exit code: ${RETURN}" |
| exit ${RETURN} |
| fi |
| done |
| # and build the dtest-jar for the branch under test |
| ${ANT_HOME}/bin/ant realclean |
| git checkout origin/$CIRCLE_BRANCH |
| git clean -fd |
| for x in $(seq 1 3); do |
| ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar |
| RETURN="$?" |
| if [ "${RETURN}" -eq "0" ]; then |
| cp build/dtest*.jar ~/dtest_jars |
| break |
| fi |
| done |
| # Exit, if we didn't build successfully |
| if [ "${RETURN}" -ne "0" ]; then |
| echo "Build failed with exit code: ${RETURN}" |
| exit ${RETURN} |
| fi |
| ls -l ~/dtest_jars |
| no_output_timeout: 15m |
| - persist_to_workspace: |
| root: /home/cassandra |
| paths: |
| - dtest_jars |
| environment: |
| - ANT_HOME: /usr/share/ant |
| - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 |
| - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - LANG: en_US.UTF-8 |
| - KEEP_TEST_DIR: true |
| - DEFAULT_DIR: /home/cassandra/cassandra-dtest |
| - PYTHONIOENCODING: utf-8 |
| - PYTHONUNBUFFERED: true |
| - CASS_DRIVER_NO_EXTENSIONS: true |
| - CASS_DRIVER_NO_CYTHON: true |
| - CASSANDRA_SKIP_SYNC: true |
| - DTEST_REPO: https://github.com/apache/cassandra-dtest.git |
| - DTEST_BRANCH: trunk |
| - CCM_MAX_HEAP_SIZE: 1024M |
| - CCM_HEAP_NEWSIZE: 256M |
| - REPEATED_TESTS_STOP_ON_FAILURE: false |
| - REPEATED_UTESTS: null |
| - REPEATED_UTESTS_COUNT: 500 |
| - REPEATED_UTESTS_FQLTOOL: null |
| - REPEATED_UTESTS_FQLTOOL_COUNT: 500 |
| - REPEATED_UTESTS_LONG: null |
| - REPEATED_UTESTS_LONG_COUNT: 100 |
| - REPEATED_UTESTS_STRESS: null |
| - REPEATED_UTESTS_STRESS_COUNT: 500 |
| - REPEATED_SIMULATOR_DTESTS: null |
| - REPEATED_SIMULATOR_DTESTS_COUNT: 500 |
| - REPEATED_JVM_DTESTS: null |
| - REPEATED_JVM_DTESTS_COUNT: 500 |
| - REPEATED_JVM_UPGRADE_DTESTS: null |
| - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 |
| - REPEATED_DTESTS: null |
| - REPEATED_DTESTS_COUNT: 500 |
| - REPEATED_LARGE_DTESTS: null |
| - REPEATED_LARGE_DTESTS_COUNT: 100 |
| - REPEATED_UPGRADE_DTESTS: null |
| - REPEATED_UPGRADE_DTESTS_COUNT: 25 |
| - REPEATED_ANT_TEST_TARGET: testsome |
| - REPEATED_ANT_TEST_CLASS: null |
| - REPEATED_ANT_TEST_METHODS: null |
| - REPEATED_ANT_TEST_VNODES: false |
| - REPEATED_ANT_TEST_COUNT: 500 |
| - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 |
| workflows: |
| version: 2 |
| java8_separate_tests: |
| jobs: |
| - start_j8_build: |
| type: approval |
| - j8_build: |
| requires: |
| - start_j8_build |
| - start_j8_unit_tests: |
| type: approval |
| - j8_unit_tests: |
| requires: |
| - start_j8_unit_tests |
| - j8_build |
| - start_j8_jvm_dtests: |
| type: approval |
| - j8_jvm_dtests: |
| requires: |
| - start_j8_jvm_dtests |
| - j8_build |
| - start_j8_jvm_dtests_vnode: |
| type: approval |
| - j8_jvm_dtests_vnode: |
| requires: |
| - start_j8_jvm_dtests_vnode |
| - j8_build |
| - start_j11_jvm_dtests: |
| type: approval |
| - j11_jvm_dtests: |
| requires: |
| - start_j11_jvm_dtests |
| - j8_build |
| - start_j11_jvm_dtests_vnode: |
| type: approval |
| - j11_jvm_dtests_vnode: |
| requires: |
| - start_j11_jvm_dtests_vnode |
| - j8_build |
| - start_j8_simulator_dtests: |
| type: approval |
| - j8_simulator_dtests: |
| requires: |
| - start_j8_simulator_dtests |
| - j8_build |
| - start_j8_cqlshlib_tests: |
| type: approval |
| - j8_cqlshlib_tests: |
| requires: |
| - start_j8_cqlshlib_tests |
| - j8_build |
| - start_j8_cqlshlib_cython_tests: |
| type: approval |
| - j8_cqlshlib_cython_tests: |
| requires: |
| - start_j8_cqlshlib_cython_tests |
| - j8_build |
| - start_j11_cqlshlib_tests: |
| type: approval |
| - j11_cqlshlib_tests: |
| requires: |
| - start_j11_cqlshlib_tests |
| - j8_build |
| - start_j11_cqlshlib_cython_tests: |
| type: approval |
| - j11_cqlshlib_cython_tests: |
| requires: |
| - start_j11_cqlshlib_cython_tests |
| - j8_build |
| - start_j11_unit_tests: |
| type: approval |
| - j11_unit_tests: |
| requires: |
| - start_j11_unit_tests |
| - j8_build |
| - start_j8_utests_long: |
| type: approval |
| - j8_utests_long: |
| requires: |
| - start_j8_utests_long |
| - j8_build |
| - start_j11_utests_long: |
| type: approval |
| - j11_utests_long: |
| requires: |
| - start_j11_utests_long |
| - j8_build |
| - start_j8_utests_cdc: |
| type: approval |
| - j8_utests_cdc: |
| requires: |
| - start_j8_utests_cdc |
| - j8_build |
| - start_j11_utests_cdc: |
| type: approval |
| - j11_utests_cdc: |
| requires: |
| - start_j11_utests_cdc |
| - j8_build |
| - start_j8_utests_compression: |
| type: approval |
| - j8_utests_compression: |
| requires: |
| - start_j8_utests_compression |
| - j8_build |
| - start_j11_utests_compression: |
| type: approval |
| - j11_utests_compression: |
| requires: |
| - start_j11_utests_compression |
| - j8_build |
| - start_j8_utests_stress: |
| type: approval |
| - j8_utests_stress: |
| requires: |
| - start_j8_utests_stress |
| - j8_build |
| - start_j11_utests_stress: |
| type: approval |
| - j11_utests_stress: |
| requires: |
| - start_j11_utests_stress |
| - j8_build |
| - start_j8_utests_fqltool: |
| type: approval |
| - j8_utests_fqltool: |
| requires: |
| - start_j8_utests_fqltool |
| - j8_build |
| - start_j11_utests_fqltool: |
| type: approval |
| - j11_utests_fqltool: |
| requires: |
| - start_j11_utests_fqltool |
| - j8_build |
| - start_j8_utests_system_keyspace_directory: |
| type: approval |
| - j8_utests_system_keyspace_directory: |
| requires: |
| - start_j8_utests_system_keyspace_directory |
| - j8_build |
| - start_j11_utests_system_keyspace_directory: |
| type: approval |
| - j11_utests_system_keyspace_directory: |
| requires: |
| - start_j11_utests_system_keyspace_directory |
| - j8_build |
| - start_j8_dtest_jars_build: |
| type: approval |
| - j8_dtest_jars_build: |
| requires: |
| - j8_build |
| - start_j8_dtest_jars_build |
| - start_jvm_upgrade_dtests: |
| type: approval |
| - j8_jvm_upgrade_dtests: |
| requires: |
| - start_jvm_upgrade_dtests |
| - j8_dtest_jars_build |
| - start_j8_dtests: |
| type: approval |
| - j8_dtests: |
| requires: |
| - start_j8_dtests |
| - j8_build |
| - start_j8_dtests_vnode: |
| type: approval |
| - j8_dtests_vnode: |
| requires: |
| - start_j8_dtests_vnode |
| - j8_build |
| - start_j8_dtests_offheap: |
| type: approval |
| - j8_dtests_offheap: |
| requires: |
| - start_j8_dtests_offheap |
| - j8_build |
| - start_j11_dtests: |
| type: approval |
| - j11_dtests: |
| requires: |
| - start_j11_dtests |
| - j8_build |
| - start_j11_dtests_vnode: |
| type: approval |
| - j11_dtests_vnode: |
| requires: |
| - start_j11_dtests_vnode |
| - j8_build |
| - start_j11_dtests_offheap: |
| type: approval |
| - j11_dtests_offheap: |
| requires: |
| - start_j11_dtests_offheap |
| - j8_build |
| - start_j8_dtests_large: |
| type: approval |
| - j8_dtests_large: |
| requires: |
| - start_j8_dtests_large |
| - j8_build |
| - start_j8_dtests_large_vnode: |
| type: approval |
| - j8_dtests_large_vnode: |
| requires: |
| - start_j8_dtests_large_vnode |
| - j8_build |
| - start_j11_dtests_large: |
| type: approval |
| - j11_dtests_large: |
| requires: |
| - start_j11_dtests_large |
| - j8_build |
| - start_j11_dtests_large_vnode: |
| type: approval |
| - j11_dtests_large_vnode: |
| requires: |
| - start_j11_dtests_large_vnode |
| - j8_build |
| - start_upgrade_dtests: |
| type: approval |
| - j8_upgrade_dtests: |
| requires: |
| - start_upgrade_dtests |
| - j8_build |
| - start_j8_cqlsh_tests: |
| type: approval |
| - j8_cqlsh_dtests_py3: |
| requires: |
| - start_j8_cqlsh_tests |
| - j8_build |
| - j8_cqlsh_dtests_py3_vnode: |
| requires: |
| - start_j8_cqlsh_tests |
| - j8_build |
| - j8_cqlsh_dtests_py38: |
| requires: |
| - start_j8_cqlsh_tests |
| - j8_build |
| - j8_cqlsh_dtests_py311: |
| requires: |
| - start_j8_cqlsh_tests |
| - j8_build |
| - j8_cqlsh_dtests_py38_vnode: |
| requires: |
| - start_j8_cqlsh_tests |
| - j8_build |
| - j8_cqlsh_dtests_py311_vnode: |
| requires: |
| - start_j8_cqlsh_tests |
| - j8_build |
| - start_j8_cqlsh_tests_offheap: |
| type: approval |
| - j8_cqlsh_dtests_py3_offheap: |
| requires: |
| - start_j8_cqlsh_tests_offheap |
| - j8_build |
| - j8_cqlsh_dtests_py38_offheap: |
| requires: |
| - start_j8_cqlsh_tests_offheap |
| - j8_build |
| - j8_cqlsh_dtests_py311_offheap: |
| requires: |
| - start_j8_cqlsh_tests_offheap |
| - j8_build |
| - start_j11_cqlsh_tests: |
| type: approval |
| - j11_cqlsh_dtests_py3: |
| requires: |
| - start_j11_cqlsh_tests |
| - j8_build |
| - j11_cqlsh_dtests_py3_vnode: |
| requires: |
| - start_j11_cqlsh_tests |
| - j8_build |
| - j11_cqlsh_dtests_py38: |
| requires: |
| - start_j11_cqlsh_tests |
| - j8_build |
| - j11_cqlsh_dtests_py311: |
| requires: |
| - start_j11_cqlsh_tests |
| - j8_build |
| - j11_cqlsh_dtests_py38_vnode: |
| requires: |
| - start_j11_cqlsh_tests |
| - j8_build |
| - j11_cqlsh_dtests_py311_vnode: |
| requires: |
| - start_j11_cqlsh_tests |
| - j8_build |
| - start_j11_cqlsh_tests_offheap: |
| type: approval |
| - j11_cqlsh_dtests_py3_offheap: |
| requires: |
| - start_j11_cqlsh_tests_offheap |
| - j8_build |
| - j11_cqlsh_dtests_py38_offheap: |
| requires: |
| - start_j11_cqlsh_tests_offheap |
| - j8_build |
| - j11_cqlsh_dtests_py311_offheap: |
| requires: |
| - start_j11_cqlsh_tests_offheap |
| - j8_build |
| java8_pre-commit_tests: |
| jobs: |
| - start_pre-commit_tests: |
| type: approval |
| - j8_build: |
| requires: |
| - start_pre-commit_tests |
| - j8_unit_tests: |
| requires: |
| - j8_build |
| - j8_simulator_dtests: |
| requires: |
| - j8_build |
| - j8_jvm_dtests: |
| requires: |
| - j8_build |
| - j8_jvm_dtests_vnode: |
| requires: |
| - j8_build |
| - j11_jvm_dtests: |
| requires: |
| - j8_build |
| - j11_jvm_dtests_vnode: |
| requires: |
| - j8_build |
| - j8_cqlshlib_tests: |
| requires: |
| - j8_build |
| - j8_cqlshlib_cython_tests: |
| requires: |
| - j8_build |
| - j11_cqlshlib_tests: |
| requires: |
| - j8_build |
| - j11_cqlshlib_cython_tests: |
| requires: |
| - j8_build |
| - j11_unit_tests: |
| requires: |
| - j8_build |
| - start_utests_long: |
| type: approval |
| - j8_utests_long: |
| requires: |
| - start_utests_long |
| - j8_build |
| - j11_utests_long: |
| requires: |
| - start_utests_long |
| - j8_build |
| - start_utests_cdc: |
| type: approval |
| - j8_utests_cdc: |
| requires: |
| - start_utests_cdc |
| - j8_build |
| - j11_utests_cdc: |
| requires: |
| - start_utests_cdc |
| - j8_build |
| - start_utests_compression: |
| type: approval |
| - j8_utests_compression: |
| requires: |
| - start_utests_compression |
| - j8_build |
| - j11_utests_compression: |
| requires: |
| - start_utests_compression |
| - j8_build |
| - start_utests_stress: |
| type: approval |
| - j8_utests_stress: |
| requires: |
| - start_utests_stress |
| - j8_build |
| - j11_utests_stress: |
| requires: |
| - start_utests_stress |
| - j8_build |
| - start_utests_fqltool: |
| type: approval |
| - j8_utests_fqltool: |
| requires: |
| - start_utests_fqltool |
| - j8_build |
| - j11_utests_fqltool: |
| requires: |
| - start_utests_fqltool |
| - j8_build |
| - start_utests_system_keyspace_directory: |
| type: approval |
| - j8_utests_system_keyspace_directory: |
| requires: |
| - j8_build |
| - j11_utests_system_keyspace_directory: |
| requires: |
| - start_utests_system_keyspace_directory |
| - j8_build |
| - start_jvm_upgrade_dtests: |
| type: approval |
| - j8_dtest_jars_build: |
| requires: |
| - j8_build |
| - start_jvm_upgrade_dtests |
| - j8_jvm_upgrade_dtests: |
| requires: |
| - j8_dtest_jars_build |
| - j8_dtests: |
| requires: |
| - j8_build |
| - j8_dtests_vnode: |
| requires: |
| - j8_build |
| - start_j8_dtests_offheap: |
| type: approval |
| - j8_dtests_offheap: |
| requires: |
| - start_j8_dtests_offheap |
| - j8_build |
| - j11_dtests: |
| requires: |
| - j8_build |
| - j11_dtests_vnode: |
| requires: |
| - j8_build |
| - start_j11_dtests_offheap: |
| type: approval |
| - j11_dtests_offheap: |
| requires: |
| - start_j11_dtests_offheap |
| - j8_build |
| - start_j8_dtests_large: |
| type: approval |
| - j8_dtests_large: |
| requires: |
| - start_j8_dtests_large |
| - j8_build |
| - j8_dtests_large_vnode: |
| requires: |
| - start_j8_dtests_large |
| - j8_build |
| - start_j11_dtests_large: |
| type: approval |
| - j11_dtests_large: |
| requires: |
| - start_j11_dtests_large |
| - j8_build |
| - j11_dtests_large_vnode: |
| requires: |
| - start_j11_dtests_large |
| - j8_build |
| - start_upgrade_dtests: |
| type: approval |
| - j8_upgrade_dtests: |
| requires: |
| - j8_build |
| - start_upgrade_dtests |
| - j8_cqlsh_dtests_py3: |
| requires: |
| - j8_build |
| - j8_cqlsh_dtests_py3_vnode: |
| requires: |
| - j8_build |
| - j8_cqlsh_dtests_py38: |
| requires: |
| - j8_build |
| - j8_cqlsh_dtests_py311: |
| requires: |
| - j8_build |
| - j8_cqlsh_dtests_py38_vnode: |
| requires: |
| - j8_build |
| - j8_cqlsh_dtests_py311_vnode: |
| requires: |
| - j8_build |
| - start_j8_cqlsh_dtests_offheap: |
| type: approval |
| - j8_cqlsh_dtests_py3_offheap: |
| requires: |
| - start_j8_cqlsh_dtests_offheap |
| - j8_build |
| - j8_cqlsh_dtests_py38_offheap: |
| requires: |
| - start_j8_cqlsh_dtests_offheap |
| - j8_build |
| - j8_cqlsh_dtests_py311_offheap: |
| requires: |
| - start_j8_cqlsh_dtests_offheap |
| - j8_build |
| - j11_cqlsh_dtests_py3: |
| requires: |
| - j8_build |
| - j11_cqlsh_dtests_py3_vnode: |
| requires: |
| - j8_build |
| - j11_cqlsh_dtests_py38: |
| requires: |
| - j8_build |
| - j11_cqlsh_dtests_py311: |
| requires: |
| - j8_build |
| - j11_cqlsh_dtests_py38_vnode: |
| requires: |
| - j8_build |
| - j11_cqlsh_dtests_py311_vnode: |
| requires: |
| - j8_build |
| - start_j11_cqlsh-dtests-offheap: |
| type: approval |
| - j11_cqlsh_dtests_py3_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j8_build |
| - j11_cqlsh_dtests_py38_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j8_build |
| - j11_cqlsh_dtests_py311_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j8_build |
| java11_separate_tests: |
| jobs: |
| - start_j11_build: |
| type: approval |
| - j11_build: |
| requires: |
| - start_j11_build |
| - start_j11_unit_tests: |
| type: approval |
| - j11_unit_tests: |
| requires: |
| - start_j11_unit_tests |
| - j11_build |
| - start_j11_jvm_dtests: |
| type: approval |
| - j11_jvm_dtests: |
| requires: |
| - start_j11_jvm_dtests |
| - j11_build |
| - start_j11_jvm_dtests_vnode: |
| type: approval |
| - j11_jvm_dtests_vnode: |
| requires: |
| - start_j11_jvm_dtests_vnode |
| - j11_build |
| - start_j11_cqlshlib_tests: |
| type: approval |
| - j11_cqlshlib_tests: |
| requires: |
| - start_j11_cqlshlib_tests |
| - j11_build |
| - start_j11_cqlshlib_cython_tests: |
| type: approval |
| - j11_cqlshlib_cython_tests: |
| requires: |
| - start_j11_cqlshlib_cython_tests |
| - j11_build |
| - start_j11_dtests: |
| type: approval |
| - j11_dtests: |
| requires: |
| - start_j11_dtests |
| - j11_build |
| - start_j11_dtests_vnode: |
| type: approval |
| - j11_dtests_vnode: |
| requires: |
| - start_j11_dtests_vnode |
| - j11_build |
| - start_j11_dtests_offheap: |
| type: approval |
| - j11_dtests_offheap: |
| requires: |
| - start_j11_dtests_offheap |
| - j11_build |
| - start_j11_dtests_large: |
| type: approval |
| - j11_dtests_large: |
| requires: |
| - start_j11_dtests_large |
| - j11_build |
| - start_j11_dtests_large_vnode: |
| type: approval |
| - j11_dtests_large_vnode: |
| requires: |
| - start_j11_dtests_large_vnode |
| - j11_build |
| - start_j11_cqlsh_tests: |
| type: approval |
| - j11_cqlsh_dtests_py3: |
| requires: |
| - start_j11_cqlsh_tests |
| - j11_build |
| - j11_cqlsh_dtests_py3_vnode: |
| requires: |
| - start_j11_cqlsh_tests |
| - j11_build |
| - j11_cqlsh_dtests_py38: |
| requires: |
| - start_j11_cqlsh_tests |
| - j11_build |
| - j11_cqlsh_dtests_py311: |
| requires: |
| - start_j11_cqlsh_tests |
| - j11_build |
| - j11_cqlsh_dtests_py38_vnode: |
| requires: |
| - start_j11_cqlsh_tests |
| - j11_build |
| - j11_cqlsh_dtests_py311_vnode: |
| requires: |
| - start_j11_cqlsh_tests |
| - j11_build |
| - start_j11_cqlsh-dtests-offheap: |
| type: approval |
| - j11_cqlsh_dtests_py3_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j11_build |
| - j11_cqlsh_dtests_py38_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j11_build |
| - j11_cqlsh_dtests_py311_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j11_build |
| - start_j11_utests_long: |
| type: approval |
| - j11_utests_long: |
| requires: |
| - start_j11_utests_long |
| - j11_build |
| - start_j11_utests_cdc: |
| type: approval |
| - j11_utests_cdc: |
| requires: |
| - start_j11_utests_cdc |
| - j11_build |
| - start_j11_utests_compression: |
| type: approval |
| - j11_utests_compression: |
| requires: |
| - start_j11_utests_compression |
| - j11_build |
| - start_j11_utests_stress: |
| type: approval |
| - j11_utests_stress: |
| requires: |
| - start_j11_utests_stress |
| - j11_build |
| - start_j11_utests_fqltool: |
| type: approval |
| - j11_utests_fqltool: |
| requires: |
| - start_j11_utests_fqltool |
| - j11_build |
| - start_j11_utests_system_keyspace_directory: |
| type: approval |
| - j11_utests_system_keyspace_directory: |
| requires: |
| - start_j11_utests_system_keyspace_directory |
| - j11_build |
| java11_pre-commit_tests: |
| jobs: |
| - start_pre-commit_tests: |
| type: approval |
| - j11_build: |
| requires: |
| - start_pre-commit_tests |
| - j11_unit_tests: |
| requires: |
| - j11_build |
| - j11_jvm_dtests: |
| requires: |
| - j11_build |
| - j11_jvm_dtests_vnode: |
| requires: |
| - j11_build |
| - j11_cqlshlib_tests: |
| requires: |
| - j11_build |
| - j11_cqlshlib_cython_tests: |
| requires: |
| - j11_build |
| - j11_dtests: |
| requires: |
| - j11_build |
| - j11_dtests_vnode: |
| requires: |
| - j11_build |
| - start_j11_dtests_offheap: |
| type: approval |
| - j11_dtests_offheap: |
| requires: |
| - start_j11_dtests_offheap |
| - j11_build |
| - start_j11_dtests_large: |
| type: approval |
| - j11_dtests_large: |
| requires: |
| - start_j11_dtests_large |
| - j11_build |
| - j11_dtests_large_vnode: |
| requires: |
| - start_j11_dtests_large |
| - j11_build |
| - j11_cqlsh_dtests_py3: |
| requires: |
| - j11_build |
| - j11_cqlsh_dtests_py3_vnode: |
| requires: |
| - j11_build |
| - j11_cqlsh_dtests_py38: |
| requires: |
| - j11_build |
| - j11_cqlsh_dtests_py311: |
| requires: |
| - j11_build |
| - j11_cqlsh_dtests_py38_vnode: |
| requires: |
| - j11_build |
| - j11_cqlsh_dtests_py311_vnode: |
| requires: |
| - j11_build |
| - start_j11_cqlsh-dtests-offheap: |
| type: approval |
| - j11_cqlsh_dtests_py3_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j11_build |
| - j11_cqlsh_dtests_py38_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j11_build |
| - j11_cqlsh_dtests_py311_offheap: |
| requires: |
| - start_j11_cqlsh-dtests-offheap |
| - j11_build |
| - start_utests_long: |
| type: approval |
| - j11_utests_long: |
| requires: |
| - start_utests_long |
| - j11_build |
| - start_utests_cdc: |
| type: approval |
| - j11_utests_cdc: |
| requires: |
| - start_utests_cdc |
| - j11_build |
| - start_utests_compression: |
| type: approval |
| - j11_utests_compression: |
| requires: |
| - start_utests_compression |
| - j11_build |
| - start_utests_stress: |
| type: approval |
| - j11_utests_stress: |
| requires: |
| - start_utests_stress |
| - j11_build |
| - start_utests_fqltool: |
| type: approval |
| - j11_utests_fqltool: |
| requires: |
| - start_utests_fqltool |
| - j11_build |
| - start_utests_system_keyspace_directory: |
| type: approval |
| - j11_utests_system_keyspace_directory: |
| requires: |
| - start_utests_system_keyspace_directory |
| - j11_build |
