system_test/system_test_runner.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#!/usr/bin/evn python

# =================================================================
# system_test_runner.py
#
# - This script is the test driver for a distributed environment
#   system testing framework. It is located at the top level of the
#   framework hierachy (in this case - system_test/).
#
# - This test driver servers as an entry point to launch a series
#   of test suites (module) with multiple functionally similar test
#   cases which can be grouped together.
#
# - Please refer to system_test/README.txt for more details on
#   how to add test suite and test case.
#
# - In most cases, it is not necessary to make any changes to this
#   script.
# =================================================================

from optparse        import OptionParser
from system_test_env import SystemTestEnv
from utils           import system_test_utils

import logging.config
import os
import pprint
import sys


# load the config file for logging
logging.config.fileConfig('logging.conf')

# 'd' is an argument to be merged into the log message (see Python doc for logging).
# In this case, corresponding class name can be appended to the end of the logging
# message to facilitate debugging.
d = {'name_of_class': '(system_test_runner)'}

def main():
    nLogger = logging.getLogger('namedLogger')
    aLogger = logging.getLogger('anonymousLogger')

    optionParser = OptionParser()
    optionParser.add_option("-p", "--print-test-descriptions-only",
                            dest="printTestDescriptionsOnly",
                            default=False,
                            action="store_true",
                            help="print test descriptions only - don't run the test")

    optionParser.add_option("-n", "--do-not-validate-remote-host",
                            dest="doNotValidateRemoteHost",
                            default=False,
                            action="store_true",
                            help="do not validate remote host (due to different kafka versions are installed)")

    (options, args) = optionParser.parse_args()

    print "\n"
    aLogger.info("=================================================")
    aLogger.info("        System Regression Test Framework")
    aLogger.info("=================================================")
    print "\n"

    testSuiteClassDictList = []

    # SystemTestEnv is a class to provide all environement settings for this session
    # such as the SYSTEM_TEST_BASE_DIR, SYSTEM_TEST_UTIL_DIR, ...
    systemTestEnv = SystemTestEnv()

    if options.printTestDescriptionsOnly:
        systemTestEnv.printTestDescriptionsOnly = True
    if options.doNotValidateRemoteHost:
        systemTestEnv.doNotValidateRemoteHost = True

    if not systemTestEnv.printTestDescriptionsOnly:
        if not systemTestEnv.doNotValidateRemoteHost:
            if not system_test_utils.setup_remote_hosts(systemTestEnv):
                nLogger.error("Remote hosts sanity check failed. Aborting test ...", extra=d)
                print
                sys.exit(1)
        else:
            nLogger.info("SKIPPING : checking remote machines", extra=d)
        print

    # get all defined names within a module: 
    definedItemList = dir(SystemTestEnv)
    aLogger.debug("=================================================")
    aLogger.debug("SystemTestEnv keys:")
    for item in definedItemList:
        aLogger.debug("    " + item)
    aLogger.debug("=================================================")

    aLogger.info("=================================================")
    aLogger.info("looking up test suites ...")
    aLogger.info("=================================================")
    # find all test suites in SYSTEM_TEST_BASE_DIR
    for dirName in os.listdir(systemTestEnv.SYSTEM_TEST_BASE_DIR):

        # make sure this is a valid testsuite directory
        if os.path.isdir(dirName) and dirName.endswith(systemTestEnv.SYSTEM_TEST_SUITE_SUFFIX):
            print
            nLogger.info("found a testsuite : " + dirName, extra=d)
            testModulePathName = os.path.abspath(systemTestEnv.SYSTEM_TEST_BASE_DIR + "/" + dirName)

            if not systemTestEnv.printTestDescriptionsOnly:
                system_test_utils.setup_remote_hosts_with_testsuite_level_cluster_config(systemTestEnv, testModulePathName)

            # go through all test modules file in this testsuite
            for moduleFileName in os.listdir(testModulePathName):

                # make sure it is a valid test module
                if moduleFileName.endswith(systemTestEnv.SYSTEM_TEST_MODULE_EXT) \
                    and not moduleFileName.startswith("__"):

                    # found a test module file
                    nLogger.info("found a test module file : " + moduleFileName, extra=d) 

                    testModuleClassName = system_test_utils.sys_call("grep ^class " + testModulePathName + "/" + \
                                          moduleFileName + " | sed 's/^class //g' | sed 's/(.*):.*//g'")
                    testModuleClassName = testModuleClassName.rstrip('\n')

                    # collect the test suite class data
                    testSuiteClassDict           = {}
                    testSuiteClassDict["suite"]  = dirName
                    extLenToRemove               = systemTestEnv.SYSTEM_TEST_MODULE_EXT.__len__() * -1 
                    testSuiteClassDict["module"] = moduleFileName[:extLenToRemove]
                    testSuiteClassDict["class"]  = testModuleClassName
                    testSuiteClassDictList.append(testSuiteClassDict)

                    suiteName  = testSuiteClassDict["suite"]
                    moduleName = testSuiteClassDict["module"]
                    className  = testSuiteClassDict["class"]

                    # add testsuite directory to sys.path such that the module can be loaded
                    sys.path.append(systemTestEnv.SYSTEM_TEST_BASE_DIR + "/" + suiteName)
 
                    if not systemTestEnv.printTestDescriptionsOnly:
                        aLogger.info("=================================================")
                        aLogger.info("Running Test for : ")
                        aLogger.info("    suite  : " + suiteName)
                        aLogger.info("    module : " + moduleName)
                        aLogger.info("    class  : " + className)
                        aLogger.info("=================================================")

                    # dynamically loading a module and starting the test class
                    mod      = __import__(moduleName)
                    theClass = getattr(mod, className)
                    instance = theClass(systemTestEnv)
                    instance.runTest()
                print

    if not systemTestEnv.printTestDescriptionsOnly:
        totalFailureCount = 0
        print
        print "========================================================"
        print "                 TEST REPORTS"
        print "========================================================"
        for systemTestResult in systemTestEnv.systemTestResultsList:
            for key in sorted(systemTestResult.iterkeys()):
                if key == "validation_status":
                    print key, " : "
                    testItemStatus = None
                    for validatedItem in sorted(systemTestResult[key].iterkeys()):
                        testItemStatus = systemTestResult[key][validatedItem]
                        print "    ", validatedItem, " : ", testItemStatus
                        if "FAILED" == testItemStatus:
                            totalFailureCount += 1
                else:
                    print key, " : ", systemTestResult[key]
            print
            print "========================================================"
            print

        print "========================================================"
        print "Total failures count : " + str(totalFailureCount)
        print "========================================================"
        print
        return totalFailureCount

    return -1

# =========================
# main entry point
# =========================

main()