Listing tests (--collect-only and --dtest-print-tests-only) only lists tests that will run according to other arguments specified
This patch fixes behaviour for both `run_dtests.py` and `pytest`.
- Error handling for invalid parameter values / combinations is in a single place (`dtest_config.py`) and is executed before we actually traverse through the tests
- We exit with just a clean error message instead of tons of spam
- `run_dtests.sh` will not loose the exit code of `pytest` any more so we can clearly detect when test cases collection fails
- removed a bit of boilerplate code from `run_dtests.py`, e.g. what it did with xml processing is simply provided with `-q` argument of `pytest`
- tests filtering has been refactored to be cleaner
- fixed filtering of resource intensive tests and other tests (note that except for upgrade tests, we took care only about test method annotations - module level annotations were ignored for vnodes, no_vnodes, no_offheap_memtables and resource_intensive, ...)
- added meta_tests for the filtering and parsing exception handling
- added special parameter --metatests which is enough to run all the meta tests
- fixed Travis configuration so that it runs meta tests
Note that now `run_dtests.py` seems to be redundant. If we need it only for listing dtests, we can simply achieve exactly the same effect using `--collect-only -q --ignore=meta_tests` arguments for `pytest` instead of `--dtest-print-tests-only`, plus we need to filter output with `grep '.py::'` (in order to not include the summary line) and pipe stdout to the target file. It is now simplified so `run_dtests.sh` just uses `pytest` with those arguments.
patch by Jacek Lewandowski; reviewed by Tomek Ĺasica and Mick Semb Wever for CASSANDRA-16399
diff --git a/.travis.yml b/.travis.yml
index 27f5ac6..3fdb3bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,21 +1,23 @@
language: python
python:
- - "2.7"
+ - "3.8"
install:
- - pip install pycodestyle==2.3.1 flake8
+ - pip install pycodestyle==2.6.0 flake8
- pip check
+ - CASS_DRIVER_NO_CYTHON=1 pip install -r requirements.txt
script:
# we want pyflakes to check all files for unused imports only
# we use flake8 because it allows us to ignore other warnings
# exclude the thrift directories - they contain auto-generated code
- - flake8 --ignore=E501,F811,F812,F822,F823,F831,F841,N8,C9 --exclude=thrift_bindings,cassandra-thrift .
+ # - flake8 --ignore=E501,F811,F812,F822,F823,F831,F841,N8,C9 --exclude=thrift_bindings,cassandra-thrift .
- git remote add apache git://github.com/apache/cassandra-dtest.git
- git fetch apache # fetch master for the next diff
# feed changed lines with no context around them to pycodestyle
# I know we don't enforce line length but if you introduce
# 200-char lines you are doing something terribly wrong.
# lint all files for everything but line length errors
- - git diff apache/master...HEAD -U0 | pycodestyle --ignore=E501 --diff
+ - git diff apache/trunk...HEAD -U0 | pycodestyle --ignore=E501 --diff
# lint all files except json_test.py for line length errors
- - git diff apache/master...HEAD -U0 | pycodestyle --diff --exclude='json_test.py' --exclude='meta_tests/assertion_test.py' --max-line-length=200
+ - git diff apache/trunk...HEAD -U0 | pycodestyle --diff --exclude='json_test.py' --exclude='meta_tests/assertion_test.py' --max-line-length=200
+ - pytest --metatests
sudo: false
diff --git a/conftest.py b/conftest.py
index 486962e..5b73dd7 100644
--- a/conftest.py
+++ b/conftest.py
@@ -26,6 +26,7 @@
logger = logging.getLogger(__name__)
+
def check_required_loopback_interfaces_available():
"""
We need at least 3 loopback interfaces configured to run almost all dtests. On Linux, loopback
@@ -37,8 +38,9 @@
if platform.system() == "Darwin":
if len(ni.ifaddresses('lo0')[AF_INET]) < 9:
pytest.exit("At least 9 loopback interfaces are required to run dtests. "
- "On Mac you can create the required loopback interfaces by running "
- "'for i in {1..9}; do sudo ifconfig lo0 alias 127.0.0.$i up; done;'")
+ "On Mac you can create the required loopback interfaces by running "
+ "'for i in {1..9}; do sudo ifconfig lo0 alias 127.0.0.$i up; done;'")
+
def pytest_addoption(parser):
parser.addoption("--use-vnodes", action="store_true", default=False,
@@ -91,14 +93,25 @@
help="Specify whether to run indev, releases, or both")
parser.addoption("--upgrade-target-version-only", action="store_true", default=False,
help="When running upgrade tests, only run tests upgrading to the current version")
+ parser.addoption("--metatests", action="store_true", default=False,
+ help="Run only meta tests")
+
+
+def pytest_configure(config):
+ """Fail fast if arguments are invalid"""
+ if not config.getoption("--help"):
+ dtest_config = DTestConfig()
+ dtest_config.setup(config)
+ if dtest_config.metatests and config.args[0] == str(os.getcwd()):
+ config.args = ['./meta_tests']
def sufficient_system_resources_for_resource_intensive_tests():
mem = virtual_memory()
- total_mem_gb = mem.total/1024/1024/1024
+ total_mem_gb = mem.total / 1024 / 1024 / 1024
logger.info("total available system memory is %dGB" % total_mem_gb)
# todo kjkj: do not hard code our bound.. for now just do 9 instances at 3gb a piece
- return total_mem_gb >= 9*3
+ return total_mem_gb >= 9 * 3
@pytest.fixture(scope='function', autouse=True)
@@ -110,6 +123,7 @@
"""
return DTestSetupOverrides()
+
@pytest.fixture(scope='function')
def fixture_dtest_cluster_name():
"""
@@ -117,17 +131,19 @@
"""
return "test"
-"""
-Not exactly sure why :\ but, this fixture needs to be scoped to function level and not
-session or class. If you invoke pytest with tests across multiple test classes, when scopped
-at session, the root logger appears to get reset between each test class invocation.
-this means that the first test to run not from the first test class (and all subsequent
-tests), will have the root logger reset and see a level of NOTSET. Scoping it at the
-class level seems to work, and I guess it's not that much extra overhead to setup the
-logger once per test class vs. once per session in the grand scheme of things.
-"""
+
@pytest.fixture(scope="function", autouse=True)
def fixture_logging_setup(request):
+ """
+ Not exactly sure why :/ but, this fixture needs to be scoped to function level and not
+ session or class. If you invoke pytest with tests across multiple test classes, when scopped
+ at session, the root logger appears to get reset between each test class invocation.
+ this means that the first test to run not from the first test class (and all subsequent
+ tests), will have the root logger reset and see a level of NOTSET. Scoping it at the
+ class level seems to work, and I guess it's not that much extra overhead to setup the
+ logger once per test class vs. once per session in the grand scheme of things.
+ """
+
# set the root logger level to whatever the user asked for
# all new loggers created will use the root logger as a template
# essentially making this the "default" active log level
@@ -283,6 +299,7 @@
os.environ.update(initial_environment)
os.environ['PYTEST_CURRENT_TEST'] = pytest_current_test
+
@pytest.fixture(scope='function')
def fixture_dtest_create_cluster_func():
"""
@@ -291,6 +308,7 @@
"""
return DTestSetup.create_ccm_cluster
+
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
def pytest_runtest_makereport(item, call):
outcome = yield
@@ -298,6 +316,7 @@
setattr(item, "rep_" + rep.when, rep)
return rep
+
@pytest.fixture(scope='function', autouse=False)
def fixture_dtest_setup(request,
dtest_config,
@@ -340,7 +359,7 @@
if len(errors) > 0:
failed = True
pytest.fail(msg='Unexpected error found in node logs (see stdout for full details). Errors: [{errors}]'
- .format(errors=str.join(", ", errors)), pytrace=False)
+ .format(errors=str.join(", ", errors)), pytrace=False)
finally:
try:
# save the logs for inspection
@@ -352,7 +371,7 @@
dtest_setup.cleanup_cluster(request)
-#Based on https://bugs.python.org/file25808/14894.patch
+# Based on https://bugs.python.org/file25808/14894.patch
def loose_version_compare(a, b):
for i, j in zip_longest(a.version, b.version, fillvalue=''):
if type(i) != type(j):
@@ -365,7 +384,7 @@
else: # i > j
return 1
- #Longer version strings with equal prefixes are equal, but if one version string is longer than it is greater
+ # Longer version strings with equal prefixes are equal, but if one version string is longer than it is greater
aLen = len(a.version)
bLen = len(b.version)
if aLen == bLen:
@@ -440,6 +459,7 @@
if skip_msg:
pytest.skip(skip_msg)
+
def _skip_ported_msg(current_running_version, ported_from_version):
if loose_version_compare(current_running_version, ported_from_version) >= 0:
return "ported to in-JVM from %s >= %s" % (ported_from_version, current_running_version)
@@ -486,6 +506,7 @@
if skip_msg:
pytest.skip(skip_msg)
+
def _skip_ported_msg(current_running_version, ported_from_version):
if loose_version_compare(current_running_version, ported_from_version) >= 0:
return "ported to in-JVM from %s >= %s" % (ported_from_version, current_running_version)
@@ -545,7 +566,7 @@
@pytest.fixture(scope='session')
def dtest_config(request):
dtest_config = DTestConfig()
- dtest_config.setup(request)
+ dtest_config.setup(request.config)
# if we're on mac, check that we have the required loopback interfaces before doing anything!
check_required_loopback_interfaces_available()
@@ -565,91 +586,88 @@
return cassandra_dir, cassandra_version
+def has_mark(item, mark):
+ if item.get_closest_marker(mark) is not None:
+ return True
+ else:
+ for item_module in inspect.getmembers(item.module, inspect.isclass):
+ if hasattr(item_module[1], "pytestmark"):
+ mark_names = [m.name for m in item_module[1].pytestmark]
+ if mark in mark_names:
+ return True
+
+ return False
+
+
+def _is_skippable(item, mark, include_marked, include_other):
+ if has_mark(item, mark):
+ if include_marked:
+ return False
+ else:
+ logger.info("SKIP: Skipping %s because it is marked with %s" % (item, mark))
+ return True
+ else:
+ if include_other:
+ return False
+ else:
+ logger.info("SKIP: Skipping %s because it is not marked with %s" % (item, mark))
+ return True
+
+
+def is_skippable(item,
+ include_upgrade_tests,
+ include_non_upgrade_tests,
+ include_resource_intensive_tests,
+ include_non_resource_intensive_tests,
+ include_vnodes_tests,
+ include_no_vnodes_tests,
+ include_no_offheap_memtables_tests):
+
+ skippable = False
+
+ skippable = skippable or _is_skippable(item, "upgrade_test", include_upgrade_tests, include_non_upgrade_tests)
+ skippable = skippable or _is_skippable(item, "resource_intensive", include_resource_intensive_tests, include_non_resource_intensive_tests)
+ skippable = skippable or _is_skippable(item, "vnodes", include_vnodes_tests, True)
+ skippable = skippable or _is_skippable(item, "no_vnodes", include_no_vnodes_tests, True)
+ skippable = skippable or _is_skippable(item, "no_offheap_memtables", include_no_offheap_memtables_tests, True)
+ skippable = skippable or _is_skippable(item, "depends_driver", False, True)
+
+ return skippable
+
+
def pytest_collection_modifyitems(items, config):
"""
This function is called upon during the pytest test collection phase and allows for modification
of the test items within the list
"""
- collect_only = config.getoption("--collect-only")
- cassandra_dir, cassandra_version = cassandra_dir_and_version(config)
- if not collect_only and cassandra_dir is None:
- if cassandra_version is None:
- raise Exception("Required dtest arguments were missing! You must provide either --cassandra-dir "
- "or --cassandra-version. You can also set 'cassandra_dir' in pytest.ini. "
- "Refer to the documentation or invoke the help with --help.")
-
- # Either cassandra_version or cassandra_dir is defined, so figure out the version
- CASSANDRA_VERSION = cassandra_version or get_version_from_build(cassandra_dir)
-
- # Check that use_off_heap_memtables is supported in this c* version
- if config.getoption("--use-off-heap-memtables") and ("3.0" <= CASSANDRA_VERSION < "3.4"):
- raise Exception("The selected Cassandra version %s doesn't support the provided option "
- "--use-off-heap-memtables, see https://issues.apache.org/jira/browse/CASSANDRA-9472 "
- "for details" % CASSANDRA_VERSION)
-
+ dtest_config = DTestConfig()
+ dtest_config.setup(config)
selected_items = []
deselected_items = []
- sufficient_system_resources_resource_intensive = sufficient_system_resources_for_resource_intensive_tests()
- logger.debug("has sufficient resources? %s" % sufficient_system_resources_resource_intensive)
+ can_run_resource_intensive_tests = dtest_config.force_execution_of_resource_intensive_tests or sufficient_system_resources_for_resource_intensive_tests()
+ if not can_run_resource_intensive_tests:
+ logger.info("Resource intensive tests will be skipped because there is not enough system resource "
+ "and --force-resource-intensive-tests was not specified")
+
+ include_upgrade_tests = dtest_config.execute_upgrade_tests or dtest_config.execute_upgrade_tests_only
+ include_non_upgrade_tests = not dtest_config.execute_upgrade_tests_only
+ include_resource_intensive_tests = can_run_resource_intensive_tests and not dtest_config.skip_resource_intensive_tests
+ include_non_resource_intensive_tests = not dtest_config.only_resource_intensive_tests
+ include_vnodes_tests = dtest_config.use_vnodes
+ include_no_vnodes_tests = not dtest_config.use_vnodes
+ include_no_offheap_memtables_tests = not dtest_config.use_off_heap_memtables
for item in items:
- deselect_test = False
-
- if config.getoption("--execute-upgrade-tests-only"):
- deselect_test = not item.get_closest_marker("upgrade_test")
- if deselect_test:
- logger.info("SKIP: Deselecting non-upgrade test %s because of --execute-upgrade-tests-only" % item.name)
-
- if item.get_closest_marker("resource_intensive") and not collect_only:
- force_resource_intensive = config.getoption("--force-resource-intensive-tests")
- skip_resource_intensive = config.getoption("--skip-resource-intensive-tests")
- if not force_resource_intensive:
- if skip_resource_intensive:
- deselect_test = True
- logger.info("SKIP: Deselecting test %s as test marked resource_intensive. To force execution of "
- "this test re-run with the --force-resource-intensive-tests command line argument" % item.name)
- if not sufficient_system_resources_resource_intensive:
- deselect_test = True
- logger.info("SKIP: Deselecting resource_intensive test %s due to insufficient system resources" % item.name)
-
- if not item.get_closest_marker("resource_intensive") and not collect_only:
- only_resource_intensive = config.getoption("--only-resource-intensive-tests")
- if only_resource_intensive:
- deselect_test = True
- logger.info("SKIP: Deselecting non resource_intensive test %s as --only-resource-intensive-tests specified" % item.name)
-
- if item.get_closest_marker("no_vnodes"):
- if config.getoption("--use-vnodes"):
- deselect_test = True
- logger.info("SKIP: Deselecting test %s as the test requires vnodes to be disabled. To run this test, "
- "re-run without the --use-vnodes command line argument" % item.name)
-
- if item.get_closest_marker("vnodes"):
- if not config.getoption("--use-vnodes"):
- deselect_test = True
- logger.info("SKIP: Deselecting test %s as the test requires vnodes to be enabled. To run this test, "
- "re-run with the --use-vnodes command line argument" % item.name)
-
- for test_item_class in inspect.getmembers(item.module, inspect.isclass):
- if not hasattr(test_item_class[1], "pytestmark"):
- continue
-
- for module_pytest_mark in test_item_class[1].pytestmark:
- if module_pytest_mark.name == "upgrade_test":
- deselect_test = not _upgrade_testing_enabled(config)
-
- if item.get_closest_marker("upgrade_test"):
- deselect_test = not _upgrade_testing_enabled(config)
-
- if item.get_closest_marker("no_offheap_memtables"):
- if config.getoption("use_off_heap_memtables"):
- deselect_test = True
-
- # deselect cqlsh tests that depend on fixing a driver behavior
- if item.get_closest_marker("depends_driver"):
- deselect_test = True
+ deselect_test = is_skippable(item,
+ include_upgrade_tests,
+ include_non_upgrade_tests,
+ include_resource_intensive_tests,
+ include_non_resource_intensive_tests,
+ include_vnodes_tests,
+ include_no_vnodes_tests,
+ include_no_offheap_memtables_tests)
if deselect_test:
deselected_items.append(item)
@@ -658,7 +676,3 @@
config.hook.pytest_deselected(items=deselected_items)
items[:] = selected_items
-
-
-def _upgrade_testing_enabled(config):
- return config.getoption("--execute-upgrade-tests") or config.getoption("--execute-upgrade-tests-only")
diff --git a/dtest_config.py b/dtest_config.py
index bb5ce8c..86e8c96 100644
--- a/dtest_config.py
+++ b/dtest_config.py
@@ -1,8 +1,13 @@
import subprocess
import os
import ccmlib.repository
+import logging
from ccmlib.common import is_win, get_version_from_build
+from pytest import UsageError
+
+logger = logging.getLogger(__name__)
+
class DTestConfig:
def __init__(self):
@@ -12,6 +17,7 @@
self.data_dir_count = -1
self.force_execution_of_resource_intensive_tests = False
self.skip_resource_intensive_tests = False
+ self.only_resource_intensive_tests = False
self.cassandra_dir = None
self.cassandra_version = None
self.cassandra_version_from_build = None
@@ -23,28 +29,66 @@
self.keep_failed_test_dir = False
self.enable_jacoco_code_coverage = False
self.jemalloc_path = find_libjemalloc()
+ self.metatests = False
- def setup(self, request):
- self.use_vnodes = request.config.getoption("--use-vnodes")
- self.use_off_heap_memtables = request.config.getoption("--use-off-heap-memtables")
- self.num_tokens = request.config.getoption("--num-tokens")
- self.data_dir_count = request.config.getoption("--data-dir-count-per-instance")
- self.force_execution_of_resource_intensive_tests = request.config.getoption("--force-resource-intensive-tests")
- self.skip_resource_intensive_tests = request.config.getoption("--skip-resource-intensive-tests")
- cassandra_dir = request.config.getoption("--cassandra-dir") or request.config.getini("cassandra_dir")
+ def setup(self, config):
+ """
+ Reads and validates configuration. Throws UsageError if configuration is invalid.
+ """
+ self.metatests = config.getoption("--metatests")
+ if self.metatests:
+ self.cassandra_dir = os.path.join(os.getcwd(), "meta_tests/cassandra-dir-4.0-beta")
+ self.cassandra_version_from_build = self.get_version_from_build()
+ return
+
+ self.use_vnodes = config.getoption("--use-vnodes")
+ self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables")
+ self.num_tokens = config.getoption("--num-tokens")
+ self.data_dir_count = config.getoption("--data-dir-count-per-instance")
+ self.force_execution_of_resource_intensive_tests = config.getoption("--force-resource-intensive-tests")
+ self.skip_resource_intensive_tests = config.getoption("--skip-resource-intensive-tests")
+ self.only_resource_intensive_tests = config.getoption("--only-resource-intensive-tests")
+ cassandra_dir = config.getoption("--cassandra-dir") or config.getini("cassandra_dir")
+ if cassandra_dir is not None and cassandra_dir.strip() == "":
+ cassandra_dir = None
if cassandra_dir is not None:
self.cassandra_dir = os.path.expanduser(cassandra_dir)
- self.cassandra_version = request.config.getoption("--cassandra-version")
+ self.cassandra_version = config.getoption("--cassandra-version")
- self.cassandra_version_from_build = self.get_version_from_build()
+ if self.cassandra_version is not None and self.cassandra_dir is not None:
+ raise UsageError("Please remove --cassandra-version because Cassandra build directory is already "
+ "defined (by --cassandra-dir or in ini file)")
- self.delete_logs = request.config.getoption("--delete-logs")
- self.execute_upgrade_tests = request.config.getoption("--execute-upgrade-tests")
- self.execute_upgrade_tests_only = request.config.getoption("--execute-upgrade-tests-only")
- self.disable_active_log_watching = request.config.getoption("--disable-active-log-watching")
- self.keep_test_dir = request.config.getoption("--keep-test-dir")
- self.keep_failed_test_dir = request.config.getoption("--keep-failed-test-dir")
- self.enable_jacoco_code_coverage = request.config.getoption("--enable-jacoco-code-coverage")
+ try:
+ self.cassandra_version_from_build = self.get_version_from_build()
+ except FileNotFoundError as fnfe:
+ raise UsageError("The Cassandra directory %s does not seem to be valid: %s" % (self.cassandra_dir, fnfe))
+
+ self.delete_logs = config.getoption("--delete-logs")
+ self.execute_upgrade_tests = config.getoption("--execute-upgrade-tests")
+ self.execute_upgrade_tests_only = config.getoption("--execute-upgrade-tests-only")
+ self.disable_active_log_watching = config.getoption("--disable-active-log-watching")
+ self.keep_test_dir = config.getoption("--keep-test-dir")
+ self.keep_failed_test_dir = config.getoption("--keep-failed-test-dir")
+ self.enable_jacoco_code_coverage = config.getoption("--enable-jacoco-code-coverage")
+
+ if self.cassandra_version is None and self.cassandra_version_from_build is None:
+ raise UsageError("Required dtest arguments were missing! You must provide either --cassandra-dir "
+ "or --cassandra-version. You can also set 'cassandra_dir' in pytest.ini. "
+ "Refer to the documentation or invoke the help with --help.")
+
+ version = self.cassandra_version or self.cassandra_version_from_build
+
+ if self.skip_resource_intensive_tests and \
+ (self.only_resource_intensive_tests or self.force_execution_of_resource_intensive_tests):
+ raise UsageError("--skip-resource-intensive-tests does not make any sense with either "
+ "--only-resource-intensive-tests or --force-resource-intensive-tests.")
+
+ # Check that use_off_heap_memtables is supported in this c* version
+ if self.use_off_heap_memtables and ("3.0" <= version < "3.4"):
+ raise UsageError("The selected Cassandra version %s doesn't support the provided option "
+ "--use-off-heap-memtables, see https://issues.apache.org/jira/browse/CASSANDRA-9472 "
+ "for details" % version)
def get_version_from_build(self):
# There are times when we want to know the C* version we're testing against
@@ -59,7 +103,6 @@
return get_version_from_build(self.cassandra_dir)
-
# Determine the location of the libjemalloc jar so that we can specify it
# through environment variables when start Cassandra. This reduces startup
# time, making the dtests run faster.
diff --git a/dtest_setup.py b/dtest_setup.py
index f613ab2..d04fb00 100644
--- a/dtest_setup.py
+++ b/dtest_setup.py
@@ -16,7 +16,7 @@
from cassandra.cluster import NoHostAvailable
from cassandra.cluster import EXEC_PROFILE_DEFAULT
from cassandra.policies import WhiteListRoundRobinPolicy
-from ccmlib.common import get_version_from_build, is_win
+from ccmlib.common import is_win
from ccmlib.cluster import Cluster
from dtest import (get_ip_from_node, make_execution_profile, get_auth_provider, get_port_from_node,
@@ -364,7 +364,7 @@
self.stop_active_log_watch()
finally:
logger.debug("removing ccm cluster {name} at: {path}".format(name=self.cluster.name,
- path=self.test_path))
+ path=self.test_path))
self.cluster.remove()
logger.debug("clearing ssl stores from [{0}] directory".format(self.test_path))
@@ -459,7 +459,6 @@
else:
logger.debug("Jacoco agent not found or is not file. Execution will not be recorded.")
-
@staticmethod
def create_ccm_cluster(dtest_setup):
logger.info("cluster ccm directory: " + dtest_setup.test_path)
@@ -520,5 +519,3 @@
version that may not be compatible with the existing configuration options
"""
self.init_default_config()
-
-
diff --git a/meta_tests/cassandra-dir-3.2/0.version.txt b/meta_tests/cassandra-dir-3.2/0.version.txt
new file mode 100644
index 0000000..4fe5631
--- /dev/null
+++ b/meta_tests/cassandra-dir-3.2/0.version.txt
@@ -0,0 +1 @@
+3.2
\ No newline at end of file
diff --git a/meta_tests/cassandra-dir-4.0-beta/0.version.txt b/meta_tests/cassandra-dir-4.0-beta/0.version.txt
new file mode 100644
index 0000000..d494f03
--- /dev/null
+++ b/meta_tests/cassandra-dir-4.0-beta/0.version.txt
@@ -0,0 +1 @@
+4.0-beta
\ No newline at end of file
diff --git a/meta_tests/cassandra-dir-4.0-beta/bin/.do-not-delete b/meta_tests/cassandra-dir-4.0-beta/bin/.do-not-delete
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/meta_tests/cassandra-dir-4.0-beta/bin/.do-not-delete
diff --git a/meta_tests/cassandra-dir-4.0-beta/conf/cassandra.yaml b/meta_tests/cassandra-dir-4.0-beta/conf/cassandra.yaml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/meta_tests/cassandra-dir-4.0-beta/conf/cassandra.yaml
diff --git a/meta_tests/conftest_test.py b/meta_tests/conftest_test.py
new file mode 100644
index 0000000..935f657
--- /dev/null
+++ b/meta_tests/conftest_test.py
@@ -0,0 +1,73 @@
+from unittest import TestCase
+
+from conftest import is_skippable
+from mock import Mock
+
+
+def _mock_responses(responses, default_response=None):
+ return lambda arg: responses[arg] if arg in responses else default_response
+
+
+def _is_skippable(item,
+ include_upgrade_tests=True,
+ include_non_upgrade_tests=True,
+ include_resource_intensive_tests=True,
+ include_non_resource_intensive_tests=True,
+ include_vnodes_tests=True,
+ include_no_vnodes_tests=True,
+ include_no_offheap_memtables_tests=True):
+ return is_skippable(item,
+ include_upgrade_tests,
+ include_non_upgrade_tests,
+ include_resource_intensive_tests,
+ include_non_resource_intensive_tests,
+ include_vnodes_tests,
+ include_no_vnodes_tests,
+ include_no_offheap_memtables_tests)
+
+
+class ConfTestTest(TestCase):
+ regular_test = Mock(name="regular_test_mock")
+ upgrade_test = Mock(name="upgrade_test_mock")
+ resource_intensive_test = Mock(name="resource_intensive_test_mock")
+ vnodes_test = Mock(name="vnodes_test_mock")
+ no_vnodes_test = Mock(name="no_vnodes_test_mock")
+ no_offheap_memtables_test = Mock(name="no_offheap_memtables_test_mock")
+ depends_driver_test = Mock(name="depends_driver_test_mock")
+
+ def setup_method(self, method):
+ self.regular_test.get_closest_marker.side_effect = _mock_responses({})
+ self.upgrade_test.get_closest_marker.side_effect = _mock_responses({"upgrade_test": True})
+ self.resource_intensive_test.get_closest_marker.side_effect = _mock_responses({"resource_intensive": True})
+ self.vnodes_test.get_closest_marker.side_effect = _mock_responses({"vnodes": True})
+ self.no_vnodes_test.get_closest_marker.side_effect = _mock_responses({"no_vnodes": True})
+ self.no_offheap_memtables_test.get_closest_marker.side_effect = _mock_responses({"no_offheap_memtables": True})
+ self.depends_driver_test.get_closest_marker.side_effect = _mock_responses({"depends_driver": True})
+
+ def test_regular_test(self):
+ assert not _is_skippable(item=self.regular_test)
+ assert _is_skippable(item=self.regular_test, include_non_upgrade_tests=False)
+ assert _is_skippable(item=self.regular_test, include_non_resource_intensive_tests=False)
+
+ def test_upgrade_test(self):
+ assert not _is_skippable(item=self.upgrade_test)
+ assert _is_skippable(item=self.upgrade_test, include_upgrade_tests=False)
+
+ def test_resource_intensive_test(self):
+ assert not _is_skippable(item=self.resource_intensive_test)
+ assert _is_skippable(item=self.resource_intensive_test, include_resource_intensive_tests=False)
+
+ def test_vnodes_test(self):
+ assert not _is_skippable(item=self.vnodes_test)
+ assert _is_skippable(item=self.vnodes_test, include_vnodes_tests=False)
+
+ def test_no_vnodes_test(self):
+ assert not _is_skippable(item=self.no_vnodes_test)
+ assert _is_skippable(item=self.no_vnodes_test, include_no_vnodes_tests=False)
+
+ def test_no_offheap_memtables_test(self):
+ assert not _is_skippable(item=self.no_offheap_memtables_test)
+ assert _is_skippable(item=self.no_offheap_memtables_test, include_no_offheap_memtables_tests=False)
+
+ def test_depends_driver_test(self):
+ assert _is_skippable(item=self.depends_driver_test)
diff --git a/meta_tests/dtest_config_test.py b/meta_tests/dtest_config_test.py
new file mode 100644
index 0000000..0a01b68
--- /dev/null
+++ b/meta_tests/dtest_config_test.py
@@ -0,0 +1,121 @@
+import os
+from re import search
+from unittest import TestCase
+
+from dtest_config import DTestConfig
+from mock import Mock, patch
+from pytest import UsageError, raises
+import ccmlib.repository
+import ccmlib.common
+
+
+def _mock_responses(responses, default_response=None):
+ return lambda input: responses[input] if input in responses else \
+ "%s/meta_tests/cassandra-dir-4.0-beta" % os.getcwd() if input == "--cassandra-dir" else default_response
+
+
+def _check_with_params(params):
+ config = Mock()
+ config.getoption.side_effect = _mock_responses(params)
+ config.getini.side_effect = _mock_responses({})
+
+ dTestConfig = DTestConfig()
+ dTestConfig.setup(config)
+ return dTestConfig
+
+
+def _check_with_params_expect(params, pattern):
+ with raises(UsageError, match=pattern):
+ _check_with_params(params)
+
+
+class DTestConfigTest(TestCase):
+
+ def test_invalid_cass_dir_no_version(self):
+ _check_with_params_expect({
+ '--cassandra-dir': 'blah'
+ }, "The Cassandra directory blah does not seem to be valid")
+
+ def test_cass_dir_and_version(self):
+ _check_with_params_expect({
+ '--cassandra-version': '3.11'
+ }, "Cassandra build directory is already defined")
+
+ def test_no_cass_dir(self):
+ with patch.object(ccmlib.repository, "setup") as mocked_setup:
+ mocked_setup.side_effect = _mock_responses({'3.2': ("%s/meta_tests/cassandra-dir-3.2" % os.getcwd(), '3.2.0')})
+ c = _check_with_params({
+ '--cassandra-dir': None,
+ '--cassandra-version': '3.2'
+ })
+ assert c.cassandra_version == '3.2'
+ assert search("^3.2", str(c.cassandra_version_from_build))
+
+ def test_valid_cass_dir_no_version(self):
+ c = _check_with_params({
+ })
+ assert c.cassandra_version is None
+ assert c.cassandra_version_from_build == '4.0-beta'
+
+ def test_no_cass_dir_no_version(self):
+ _check_with_params_expect({
+ '--cassandra-dir': None
+ }, "You must provide either --cassandra-dir or --cassandra-version")
+
+ def test_illegal_args_combinations_for_resource_intensive_tests(self):
+ _check_with_params_expect({
+ '--only-resource-intensive-tests': True,
+ '--skip-resource-intensive-tests': True
+ }, 'does not make any sense')
+
+ _check_with_params_expect({
+ '--force-resource-intensive-tests': True,
+ '--skip-resource-intensive-tests': True
+ }, 'does not make any sense')
+
+ _check_with_params_expect({
+ '--only-resource-intensive-tests': True,
+ '--force-resource-intensive-tests': True,
+ '--skip-resource-intensive-tests': True
+ }, 'does not make any sense')
+
+ def test_legal_args_combinations_for_resource_intensive_tests(self):
+ c = _check_with_params({
+ '--only-resource-intensive-tests': True
+ })
+ assert c.only_resource_intensive_tests
+ assert not c.skip_resource_intensive_tests
+ assert not c.force_execution_of_resource_intensive_tests
+
+ c = _check_with_params({
+ '--only-resource-intensive-tests': True,
+ '--force-resource-intensive-tests': True
+ })
+ assert c.only_resource_intensive_tests
+ assert not c.skip_resource_intensive_tests
+ assert c.force_execution_of_resource_intensive_tests
+
+ c = _check_with_params({
+ '--skip-resource-intensive-tests': True
+ })
+ assert not c.only_resource_intensive_tests
+ assert c.skip_resource_intensive_tests
+ assert not c.force_execution_of_resource_intensive_tests
+
+ c = _check_with_params({
+ })
+ assert not c.only_resource_intensive_tests
+ assert not c.skip_resource_intensive_tests
+ assert not c.force_execution_of_resource_intensive_tests
+
+ def off_heap_memtables_not_supported(self):
+ _check_with_params_expect({
+ '--cassandra-dir': "%s/meta_tests/cassandra-dir-3.2" % os.getcwd(),
+ '--use-off-heap-memtables': True
+ }, "The selected Cassandra version 3.2 doesn't support the provided option")
+
+ def off_heap_memtables_supported(self):
+ c = _check_with_params({
+ '--use-off-heap-memtables': True
+ })
+ assert c.use_off_heap_memtables
diff --git a/run_dtests.py b/run_dtests.py
index 44969b2..34dd5af 100755
--- a/run_dtests.py
+++ b/run_dtests.py
@@ -1,20 +1,21 @@
#!/usr/bin/env python
"""
-usage: run_dtests.py [-h] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens NUM_TOKENS] [--data-dir-count-per-instance DATA_DIR_COUNT_PER_INSTANCE] [--force-resource-intensive-tests]
- [--skip-resource-intensive-tests] [--cassandra-dir CASSANDRA_DIR] [--cassandra-version CASSANDRA_VERSION] [--delete-logs] [--execute-upgrade-tests] [--execute-upgrade-tests-only] [--disable-active-log-watching]
- [--keep-test-dir] [--enable-jacoco-code-coverage] [--dtest-enable-debug-logging] [--dtest-print-tests-only] [--dtest-print-tests-output DTEST_PRINT_TESTS_OUTPUT]
- [--pytest-options PYTEST_OPTIONS] [--dtest-tests DTEST_TESTS]
+usage: run_dtests.py [-h] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens=NUM_TOKENS] [--data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE]
+ [--force-resource-intensive-tests] [--skip-resource-intensive-tests] [--cassandra-dir=CASSANDRA_DIR] [--cassandra-version=CASSANDRA_VERSION]
+ [--delete-logs] [--execute-upgrade-tests] [--execute-upgrade-tests-only] [--disable-active-log-watching] [--keep-test-dir]
+ [--enable-jacoco-code-coverage] [--dtest-enable-debug-logging] [--dtest-print-tests-only] [--dtest-print-tests-output=DTEST_PRINT_TESTS_OUTPUT]
+ [--pytest-options=PYTEST_OPTIONS] [--dtest-tests=DTEST_TESTS]
optional arguments:
-h, --help show this help message and exit
--use-vnodes Determines wither or not to setup clusters using vnodes for tests (default: False)
--use-off-heap-memtables Enable Off Heap Memtables when creating test clusters for tests (default: False)
- --num-tokens NUM_TOKENS Number of tokens to set num_tokens yaml setting to when creating instances with vnodes enabled (default: 256)
- --data-dir-count-per-instance DATA_DIR_COUNT_PER_INSTANCE Control the number of data directories to create per instance (default: 3)
+ --num-tokens=NUM_TOKENS Number of tokens to set num_tokens yaml setting to when creating instances with vnodes enabled (default: 256)
+ --data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE Control the number of data directories to create per instance (default: 3)
--force-resource-intensive-tests Forces the execution of tests marked as resource_intensive (default: False)
--skip-resource-intensive-tests Skip all tests marked as resource_intensive (default: False)
- --cassandra-dir CASSANDRA_DIR
- --cassandra-version CASSANDRA_VERSION
+ --cassandra-dir=CASSANDRA_DIR
+ --cassandra-version=CASSANDRA_VERSION
--delete-logs
--execute-upgrade-tests Execute Cassandra Upgrade Tests (e.g. tests annotated with the upgrade_test mark) (default: False)
--execute-upgrade-tests-only Execute Cassandra Upgrade Tests without running any other tests (e.g. tests annotated with the upgrade_test mark) (default: False)
@@ -24,9 +25,9 @@
--enable-jacoco-code-coverage Enable JaCoCo Code Coverage Support (default: False)
--dtest-enable-debug-logging Enable debug logging (for this script, pytest, and during execution of test functions) (default: False)
--dtest-print-tests-only Print list of all tests found eligible for execution given the provided options. (default: False)
- --dtest-print-tests-output DTEST_PRINT_TESTS_OUTPUT Path to file where the output of --dtest-print-tests-only should be written to (default: False)
- --pytest-options PYTEST_OPTIONS Additional command line arguments to proxy directly thru when invoking pytest. (default: None)
- --dtest-tests DTEST_TESTS Comma separated list of test files, test classes, or test methods to execute. (default: None)
+ --dtest-print-tests-output=DTEST_PRINT_TESTS_OUTPUT Path to file where the output of --dtest-print-tests-only should be written to (default: False)
+ --pytest-options=PYTEST_OPTIONS Additional command line arguments to proxy directly thru when invoking pytest. (default: None)
+ --dtest-tests=DTEST_TESTS Comma separated list of test files, test classes, or test methods to execute. (default: None)
"""
import subprocess
import sys
@@ -36,13 +37,11 @@
from os import getcwd
from tempfile import NamedTemporaryFile
-from bs4 import BeautifulSoup
from _pytest.config.argparsing import Parser
import argparse
from conftest import pytest_addoption
-from ccmlib.common import get_version_from_build
logger = logging.getLogger(__name__)
@@ -87,19 +86,6 @@
args = parser.parse_args()
- if not args.dtest_print_tests_only:
- if args.cassandra_dir is None and args.cassandra_version is None:
- raise Exception("Required dtest arguments were missing! You must provide either --cassandra-dir "
- "or --cassandra-version. Refer to the documentation or invoke the help with --help.")
-
- # Either cassandra_version or cassandra_dir is defined, so figure out the version
- CASSANDRA_VERSION = args.cassandra_version or get_version_from_build(args.cassandra_dir)
-
- if args.use_off_heap_memtables and ("3.0" <= CASSANDRA_VERSION < "3.4"):
- raise Exception("The selected Cassandra version %s doesn't support the provided option "
- "--use-off-heap-memtables, see https://issues.apache.org/jira/browse/CASSANDRA-9472 "
- "for details" % CASSANDRA_VERSION)
-
if args.dtest_enable_debug_logging:
logging.root.setLevel(logging.DEBUG)
logger.setLevel(logging.DEBUG)
@@ -111,7 +97,6 @@
handler.setFormatter(formatter)
logging.root.addHandler(handler)
-
# Get dictionaries corresponding to each point in the configuration matrix
# we want to run, then generate a config object for each of them.
logger.debug('Generating configurations from the following matrix:\n\t{}'.format(args))
@@ -125,11 +110,14 @@
if args.dtest_print_tests_only:
args_to_invoke_pytest.append("'--collect-only'")
+ args_to_invoke_pytest.append("'-q'")
if args.dtest_tests:
for test in args.dtest_tests.split(","):
args_to_invoke_pytest.append("'{test_name}'".format(test_name=test))
+ args_to_invoke_pytest.append("'--ignore=meta_tests'")
+
original_raw_cmd_args = ", ".join(args_to_invoke_pytest)
logger.debug("args to call with: [%s]" % original_raw_cmd_args)
@@ -139,14 +127,12 @@
# but for now just leaving it as is, because it does the job (although
# certainly is still pretty complicated code and has a hacky feeling)
to_execute = (
- "import pytest\n" +
- (
- "pytest.main([{options}])\n").format(options=original_raw_cmd_args)
- )
+ "import pytest\n"
+ "import sys\n"
+ "sys.exit(pytest.main([{options}]))\n".format(options=original_raw_cmd_args))
temp = NamedTemporaryFile(dir=getcwd())
logger.debug('Writing the following to {}:'.format(temp.name))
- logger.debug('```\n{to_execute}```\n'.format(to_execute=to_execute))
temp.write(to_execute.encode("utf-8"))
temp.flush()
@@ -163,20 +149,19 @@
if args.dtest_print_tests_only:
stdout, stderr = sp.communicate()
- if stderr:
+ if sp.returncode != 0:
print(stderr.decode("utf-8"))
result = sp.returncode
exit(result)
all_collected_test_modules = collect_test_modules(stdout)
joined_test_modules = "\n".join(all_collected_test_modules)
- #print("Collected %d Test Modules" % len(all_collected_test_modules))
+ print("Collected %d Test Modules" % len(all_collected_test_modules))
if args.dtest_print_tests_output is not None:
collected_tests_output_file = open(args.dtest_print_tests_output, "w")
collected_tests_output_file.write(joined_test_modules)
collected_tests_output_file.close()
- print(joined_test_modules)
else:
while True:
stdout_output = sp.stdout.readline()
@@ -197,96 +182,14 @@
def collect_test_modules(stdout):
- """
- Takes the xml-ish (no, it's not actually xml so we need to format it a bit) --collect-only output as printed
- by pytest to stdout and normalizes it to get a list of all collected tests in a human friendly format
- :param stdout: the stdout from pytest (should have been invoked with the --collect-only cmdline argument)
- :return: a formatted list of collected test modules in format test_file.py::TestClass::test_function
- """
- # unfortunately, pytest emits xml like output -- but it's not actually xml, so we'll fail to parse
- # if we try. first step is to fix up the pytest output to create well formatted xml
- xml_line_regex_pattern = re.compile(r"^([\s])*<(Module|Class|Function|Instance) '(.*)'>")
- is_first_module = True
- is_first_class = True
- has_closed_class = False
- section_has_instance = False
- section_has_class = False
- test_collect_xml_lines = []
-
- test_collect_xml_lines.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
- test_collect_xml_lines.append("<Modules>")
- for line in stdout.decode("utf-8").split('\n'):
- re_ret = re.search(xml_line_regex_pattern, line)
- if re_ret:
- if not is_first_module and re_ret.group(2) == "Module":
- if section_has_instance:
- test_collect_xml_lines.append(" </Instance>")
- if section_has_class:
- test_collect_xml_lines.append(" </Class>")
-
- test_collect_xml_lines.append(" </Module>")
- is_first_class = True
- has_closed_class = False
- section_has_instance = False
- section_has_class = False
- is_first_module = False
- elif is_first_module and re_ret.group(2) == "Module":
- if not has_closed_class and section_has_instance:
- test_collect_xml_lines.append(" </Instance>")
- if not has_closed_class and section_has_class:
- test_collect_xml_lines.append(" </Class>")
-
- is_first_class = True
- is_first_module = False
- has_closed_class = False
- section_has_instance = False
- section_has_class = False
- elif re_ret.group(2) == "Instance":
- section_has_instance = True
- elif not is_first_class and re_ret.group(2) == "Class":
- if section_has_instance:
- test_collect_xml_lines.append(" </Instance>")
- if section_has_class:
- test_collect_xml_lines.append(" </Class>")
- has_closed_class = True
- section_has_class = True
- elif re_ret.group(2) == "Class":
- is_first_class = False
- section_has_class = True
- has_closed_class = False
-
- if re_ret.group(2) == "Function":
- test_collect_xml_lines.append(" <Function name=\"{name}\"></Function>"
- .format(name=re_ret.group(3)))
- elif re_ret.group(2) == "Class":
- test_collect_xml_lines.append(" <Class name=\"{name}\">".format(name=re_ret.group(3)))
- elif re_ret.group(2) == "Module":
- test_collect_xml_lines.append(" <Module name=\"{name}\">".format(name=re_ret.group(3)))
- elif re_ret.group(2) == "Instance":
- test_collect_xml_lines.append(" <Instance name=\"\">".format(name=re_ret.group(3)))
- else:
- test_collect_xml_lines.append(line)
-
- test_collect_xml_lines.append(" </Instance>")
- test_collect_xml_lines.append(" </Class>")
- test_collect_xml_lines.append(" </Module>")
- test_collect_xml_lines.append("</Modules>")
-
+ test_regex_pattern = re.compile(r".+::.+::.+")
all_collected_test_modules = []
-
- # parse the now valid xml
- print("\n".join(test_collect_xml_lines))
- test_collect_xml = BeautifulSoup("\n".join(test_collect_xml_lines), "lxml-xml")
-
- # find all Modules (followed by classes in those modules, and then finally functions)
- for pytest_module in test_collect_xml.findAll("Module"):
- for test_class_name in pytest_module.findAll("Class"):
- for function_name in test_class_name.findAll("Function"):
- # adds to test list in format like test_file.py::TestClass::test_function for every test function found
- all_collected_test_modules.append("{module_name}::{class_name}::{function_name}"
- .format(module_name=pytest_module.attrs['name'],
- class_name=test_class_name.attrs['name'],
- function_name=function_name.attrs['name']))
+ for line in stdout.decode("utf-8").split('\n'):
+ re_ret = re.search(test_regex_pattern, line)
+ if re_ret:
+ all_collected_test_modules.append(line)
+ elif line.strip() != "":
+ print(line)
return all_collected_test_modules
