blob: 03c2e473418c2ecc4f4c2d20497e8ce36f8b1c3c [file] [log] [blame]
#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import argparse
from copy import copy
from glob import glob
import multiprocessing
from multiprocessing.pool import ThreadPool
import subprocess
import sys
import threading
import textwrap
import os
CPUS = multiprocessing.cpu_count()
CONCURRENT_SUITES = (CPUS // 4) or CPUS
CONCURRENT_TESTS = CPUS // CONCURRENT_SUITES
PROC_TIMEOUT = 360
ALT_PKG_PATHS = {
'Allura': 'allura/tests/',
}
NOT_MULTIPROC_SAFE = [
'ForgeGit',
'ForgeSVN',
]
def run_one(cmd, **popen_kwargs):
cmd_to_show = u'`{}` in {}'.format(cmd, popen_kwargs.get('cwd', '.'))
print u'{} running {}\n'.format(threading.current_thread(), cmd_to_show)
sys.stdout.flush()
all_popen_kwargs = dict(shell=True, stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
bufsize=1, # 1 == line-buffered
close_fds='posix' in sys.builtin_module_names)
all_popen_kwargs.update(popen_kwargs)
proc = subprocess.Popen(cmd, **all_popen_kwargs)
while proc.poll() is None:
line = proc.stdout.readline()
sys.stdout.write(line)
sys.stdout.flush()
# wait for completion and get remainder of output
out_remainder, _ = proc.communicate()
sys.stdout.write(out_remainder)
sys.stdout.flush()
print u'finished {}'.format(cmd_to_show)
sys.stdout.flush()
return proc
def run_many(cmds, processes=None):
"""
cmds: list of shell commands, or list of (shell cmds, popen_kwargs)
processes: number of processes, or None for # of CPU cores
"""
thread_pool = ThreadPool(processes=processes)
async_results = []
for cmd_kwds in cmds:
if type(cmd_kwds) == ():
cmd = cmd_kwds
kwds = {}
else:
cmd = cmd_kwds[0]
kwds = cmd_kwds[1]
result = thread_pool.apply_async(run_one, args=(cmd,), kwds=kwds)
async_results.append(result)
thread_pool.close()
thread_pool.join()
procs = [async_result.get() for async_result in async_results]
return [p.returncode for p in procs]
def get_packages():
packages = sorted([p.split('/')[0] for p in glob("*/setup.py")])
# make it first, to catch syntax errors
packages.remove('AlluraTest')
packages.insert(0, 'AlluraTest')
return packages
def check_packages(packages):
for pkg in packages:
try:
__import__(pkg.lower())
except ImportError:
print "Not running tests for {}, since it isn't set up".format(pkg)
else:
yield pkg
def run_tests_in_parallel(options, nosetests_args):
# coverage and multiproc plugins not compatible
use_multiproc = '--with-coverage' not in nosetests_args
def get_pkg_path(pkg):
return ALT_PKG_PATHS.get(pkg, '')
def get_multiproc_args(pkg):
if not use_multiproc or options.concurrent_tests == 1:
return ''
return ('--processes={procs_per_suite} --process-timeout={proc_timeout}'.format(
procs_per_suite=options.concurrent_tests,
proc_timeout=PROC_TIMEOUT)
if pkg not in NOT_MULTIPROC_SAFE else '')
def get_concurrent_suites():
if use_multiproc or '-n' in sys.argv:
return options.concurrent_suites
return CPUS
cmds = []
env = dict(os.environ,
NOSE_IGNORE_CONFIG_FILES='1')
for package in check_packages(options.packages):
cover_package = package.lower()
our_nosetests_args = copy(nosetests_args)
our_nosetests_args.append('--cover-package={}'.format(cover_package))
cmd = "nosetests {pkg_path} {nosetests_args} {multiproc_args}".format(
pkg_path=get_pkg_path(package),
nosetests_args=' '.join(our_nosetests_args),
multiproc_args=get_multiproc_args(package),
)
cmds.append((cmd, dict(cwd=package, env=env)))
# TODO: add a way to include this or not; and add xml output for Jenkins
cmds.append(('npm run lint-es6', {}))
return run_many(cmds, processes=get_concurrent_suites())
def parse_args():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=textwrap.dedent('''
All additional arguments are passed along to nosetests
(e.g. -v --with-coverage)
Note: --cover-package will be set automatically to the appropriate value'''))
parser.add_argument('-n', help='Number of test suites to run concurrently in separate '
'processes. Default: # CPUs / 4',
dest='concurrent_suites', type=int, default=CONCURRENT_SUITES)
parser.add_argument('-m', help='Number of tests to run concurrently in separate '
'processes, per suite. Default: # CPUs / # concurrent suites',
dest='concurrent_tests', type=int, default=CONCURRENT_TESTS)
parser.add_argument(
'-p', help='List of packages to run tests on. Default: all',
dest='packages', choices=get_packages(), default=get_packages(),
nargs='+')
return parser.parse_known_args()
if __name__ == "__main__":
ret_codes = run_tests_in_parallel(*parse_args())
sys.exit(any(ret_codes))