run_dtests.py

#!/usr/bin/env python
"""
Usage: run_dtests.py [--nose-options NOSE_OPTIONS] [TESTS...] [--vnodes VNODES_OPTIONS...]
                 [--runner-debug | --runner-quiet] [--dry-run]

nosetests options:
    --nose-options NOSE_OPTIONS  specify options to pass to `nosetests`.
    TESTS                        space-separated list of tests to pass to `nosetests`

script configuration options:
    --runner-debug -d            print debug statements in this script
    --runner-quiet -q            quiet all output from this script

cluster configuration options:
    --vnodes VNODES_OPTIONS...   specify whether to run with or without vnodes.
                                 valid values: 'true' and 'false'

example:
    The following command will execute nosetests with the '-v' (verbose) option, vnodes disabled, and run a single test:
    ./run_dtests.py --nose-options -v --vnodes false repair_tests/repair_test.py:TestRepair.token_range_repair_test_with_cf

"""
from __future__ import print_function

import subprocess
from collections import namedtuple
from itertools import product
from os import getcwd
from tempfile import NamedTemporaryFile

from docopt import docopt

from plugins.dtestconfig import GlobalConfigObject


# Generate values in a matrix from these lists of values for each attribute
# not defined in arguments to the runner script.
default_config_matrix = GlobalConfigObject(
    vnodes=(True, False),
)


def _noop(*args, **kwargs):
    pass


class ValidationResult(namedtuple('_ValidationResult', ['serialized', 'error_messages'])):
    """
    A value to be returned from validation functions. If serialization works,
    return one with 'serialized' set, otherwise return a list of string on the
    'error_messages' attribute.
    """
    __slots__ = ()

    def __new__(cls, serialized=None, error_messages=None):
        if error_messages is None:
            error_messages = []

        success_result = serialized is not None
        failure_result = bool(error_messages)

        if success_result + failure_result != 1:
            msg = ('attempted to instantiate a {cls_name} with serialized='
                   '{serialized} and error_messages={error_messages}. {cls_name} '
                   'objects must be instantiated with either a serialized or '
                   'error_messages argument, but not both.')
            msg = msg.format(cls_name=cls.__name__,
                             serialized=serialized,
                             error_messages=error_messages)
            raise ValueError(msg)

        return super(ValidationResult, cls).__new__(cls, serialized=serialized, error_messages=error_messages)


def _validate_and_serialize_vnodes(vnodes_value):
    """
    Validate the values received for vnodes configuration. Returns a
    ValidationResult.

    If the values validate, return a ValidationResult with 'serialized' set to
    the equivalent of:

        tuple(set({'true': True, 'false':False}[v.lower()] for v in vnodes_value))

    If the values don't validate, return a ValidationResult with 'messages' set
    to a list of strings, each of which points out an invalid value.
    """
    messages = []
    vnodes_value = set(v.lower() for v in vnodes_value)
    value_map = {'true': True, 'false': False}

    for v in vnodes_value:
        if v not in value_map:
            messages.append('{} not a valid value for --vnodes option. '
                            'valid values are {} (case-insensitive)'.format(v, ', '.join(list(value_map))))

    if messages:
        return ValidationResult(error_messages=messages)

    serialized = tuple({value_map[v] for v in vnodes_value})
    return ValidationResult(serialized=serialized)


def validate_and_serialize_options(docopt_options):
    """
    For each value that should be configured for a config object, attempt to
    serialize the passed-in strings into objects that can be used for
    configuration. If no values were passed in, use the list of options from
    the defaults above.

    Raises a ValueError and prints an error message if any values are invalid
    or didn't serialize correctly.
    """
    vnodes = _validate_and_serialize_vnodes(docopt_options['--vnodes'])
    if vnodes.error_messages:
        raise ValueError('Validation error:\n{}'.format('\t\n'.join(list(vnodes.error_messages))))
    return GlobalConfigObject(
        vnodes=vnodes.serialized or default_config_matrix.vnodes
    )


def product_of_values(d):
    """
    Transforms a dictionary of {key: list(configuration_options} into a tuple
    of dictionaries, each corresponding to a point in the product, with the
    values preserved at the keys where they were found in the argument.

    This is difficult to explain and is probably best demonstrated with an
    example:

        >>> from pprint import pprint
        >>> from runner import product_of_values
        >>> pprint(product_of_values(
        ...     {'a': [1, 2, 3],
        ...      'b': [4, 5, 6]}
        ... ))
        ({'a': 1, 'b': 4},
         {'a': 1, 'b': 5},
         {'a': 1, 'b': 6},
         {'a': 2, 'b': 4},
         {'a': 2, 'b': 5},
         {'a': 2, 'b': 6},
         {'a': 3, 'b': 4},
         {'a': 3, 'b': 5},
         {'a': 3, 'b': 6})

    So, in this case, we get something like

        for a_value in d['a']:
            for b_value in d['b']:
                yield {'a': a_value, 'b': b_value}

    This method does that, but for dictionaries with arbitrary iterables at
    arbitrary numbers of keys.
    """

    # transform, e.g., {'a': [1, 2, 3], 'b': [4, 5, 6]} into
    # [[('a', 1), ('a', 2), ('a', 3)],
    #  [('b', 4), ('b', 5), ('b', 6)]]
    tuple_list = [[(k, v) for v in v_list] for k, v_list in d.items()]

    # return the cartesian product of the flattened dict
    return tuple(dict(result) for result in product(*tuple_list))


if __name__ == '__main__':
    options = docopt(__doc__)
    validated_options = validate_and_serialize_options(options)

    nose_options = options['--nose-options'] or ''
    nose_option_list = nose_options.split()
    test_list = options['TESTS']
    nose_argv = nose_option_list + test_list

    verbosity = 1  # default verbosity level
    if options['--runner-debug']:
        verbosity = 2
    if options['--runner-quiet']:  # --debug and --quiet are mutually exclusive, enforced by docopt
        verbosity = 0

    debug = print if verbosity >= 2 else _noop
    output = print if verbosity >= 1 else _noop

    # Get dictionaries corresponding to each point in the configuration matrix
    # we want to run, then generate a config object for each of them.
    debug('Generating configurations from the following matrix:\n\t{}'.format(validated_options))
    all_configs = tuple(GlobalConfigObject(**d) for d in
                        product_of_values(validated_options._asdict()))
    output('About to run nosetests with config objects:\n'
           '\t{configs}\n'.format(configs='\n\t'.join(map(repr, all_configs))))

    results = []
    for config in all_configs:
        # These properties have to hold if we want to evaluate their reprs
        # below in the generated file.
        assert eval(repr(config), {'GlobalConfigObject': GlobalConfigObject}, {}) == config
        assert eval(repr(nose_argv), {}, {}) == nose_argv

        output('Running dtests with config object {}'.format(config))

        # Generate a file that runs nose, passing in config as the
        # configuration object.
        #
        # Yes, this is icky. The reason we do it is because we're dealing with
        # global configuration. We've decided global, nosetests-run-level
        # configuration is the way to go. This means we don't want to call
        # nose.main() multiple times in the same Python interpreter -- I have
        # not yet found a way to re-execute modules (thus getting new
        # module-level configuration) for each call. This didn't even work for
        # me with exec(script, {}, {}). So, here we are.
        #
        # How do we execute code in a new interpreter each time? Generate the
        # code as text, then shell out to a new interpreter.
        to_execute = (
            'import nose\n'
            'from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n'
            'nose.main(addplugins=[DtestConfigPlugin({config})])\n'
        ).format(config=repr(config))
        temp = NamedTemporaryFile(dir=getcwd())
        debug('Writing the following to {}:'.format(temp.name))

        debug('```\n{to_execute}```\n'.format(to_execute=to_execute))
        temp.write(to_execute)
        temp.flush()

        # We pass nose_argv as options to the python call to maintain
        # compatibility with the nosetests command. Arguments passed in via the
        # command line are treated one way, args passed in as
        # nose.main(argv=...) are treated another. Compare with the options
        # -xsv for an example.
        cmd_list = ['python', temp.name] + nose_argv
        debug('subprocess.call-ing {cmd_list}'.format(cmd_list=cmd_list))

        if options['--dry-run']:
            print('Would run the following command:\n\t{}'.format(cmd_list))
            with open(temp.name, 'r') as f:
                contents = f.read()
            print('{temp_name} contains:\n```\n{contents}```\n'.format(
                temp_name=temp.name,
                contents=contents
            ))
        else:
            results.append(subprocess.call(cmd_list))
        # separate the end of the last subprocess.call output from the
        # beginning of the next by printing a newline.
        print()

    # If this answer:
    # http://stackoverflow.com/a/21788998/3408454
    # is to be believed, nosetests will exit with 0 on success, 1 on test or
    # other failure, and 2 on printing usage. We'll just grab the max of the
    # runs we saw -- if one printed usage, the whole run "printed usage", if
    # none printed usage, and one or more failed, we failed, else success.
    if not results:
        results = [0]
    exit(max(results))