| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import http.client |
| import os |
| import re |
| import shutil |
| import ssl |
| import subprocess |
| import sys |
| import time |
| import traceback |
| import urllib.error |
| import urllib.request |
| from textwrap import dedent |
| |
| # Example: Checking out Revision e441a99009a557f82ea17ee9f9c3e9b89c75cee6 (refs/remotes/origin/master) |
| reGitRev = re.compile(r'Checking out Revision (\S+)\s+\(refs/remotes/origin/([^)]+)') |
| |
| # Policeman Jenkins example: [Lucene-Solr-7.x-Linux] $ /var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/bin/ant "-Dargs=-XX:-UseCompressedOops -XX:+UseConcMarkSweepGC" jenkins-hourly |
| # Policeman Jenkins Windows example: [Lucene-Solr-master-Windows] $ cmd.exe /C "C:\Users\jenkins\tools\hudson.tasks.Ant_AntInstallation\ANT_1.8.2\bin\ant.bat '"-Dargs=-client -XX:+UseConcMarkSweepGC"' jenkins-hourly && exit %%ERRORLEVEL%%" |
| # ASF Jenkins example: [Lucene-Solr-Tests-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant jenkins-hourly |
| # ASF Jenkins nightly example: [checkout] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant -file build.xml -Dtests.multiplier=2 -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt jenkins-nightly |
| # ASF Jenkins smoker example: [Lucene-Solr-SmokeRelease-master] $ /home/jenkins/tools/ant/apache-ant-1.8.4/bin/ant nightly-smoke |
| reAntInvocation = re.compile(r'\bant(?:\.bat)?\s+.*(?:jenkins-(?:hourly|nightly)|nightly-smoke)') |
| reAntSysprops = re.compile(r'"-D[^"]+"|-D[^=]+="[^"]*"|-D\S+') |
| |
| # Method example: NOTE: reproduce with: ant test -Dtestcase=ZkSolrClientTest -Dtests.method=testMultipleWatchesAsync -Dtests.seed=6EF5AB70F0032849 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=NST -Dtests.asserts=true -Dtests.file.encoding=UTF-8 |
| # Suite example: NOTE: reproduce with: ant test -Dtestcase=CloudSolrClientTest -Dtests.seed=DB2DF2D8228BAF27 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es-AR -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII |
| reReproLine = re.compile(r'NOTE:\s+reproduce\s+with:(\s+ant\s+test\s+-Dtestcase=(\S+)\s+(?:-Dtests.method=\S+\s+)?(.*))') |
| reTestsSeed = re.compile(r'-Dtests.seed=\S+\s*') |
| |
| # Example: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/21108/ |
| reJenkinsURLWithoutConsoleText = re.compile(r'https?://.*/\d+/?\Z', re.IGNORECASE) |
| |
| reJavaFile = re.compile(r'(.*)\.java\Z') |
| reModule = re.compile(r'\.[\\/](.*)[\\/]src[\\/]') |
| reTestOutputFile = re.compile(r'TEST-(.*\.([^-.]+))(?:-\d+)?\.xml\Z') |
| reErrorFailure = re.compile(r'(?:errors|failures)="[^0]') |
| reGitMainBranch = re.compile(r'^(?:master|branch_[x_\d]+)$') |
| |
| # consoleText from Policeman Jenkins's Windows jobs fails to decode as UTF-8 |
| encoding = 'iso-8859-1' |
| |
| lastFailureCode = 0 |
| gitCheckoutSucceeded = False |
| |
| description = dedent('''\ |
| Must be run from a Lucene/Solr git workspace. Downloads the Jenkins |
| log pointed to by the given URL, parses it for Git revision and failed |
| Lucene/Solr tests, checks out the Git revision in the local workspace, |
| groups the failed tests by module, then runs |
| 'ant test -Dtest.dups=%d -Dtests.class="*.test1[|*.test2[...]]" ...' |
| in each module of interest, failing at the end if any of the runs fails. |
| To control the maximum number of concurrent JVMs used for each module's |
| test run, set 'tests.jvms', e.g. in ~/lucene.build.properties |
| ''') |
| defaultIters = 5 |
| |
| def readConfig(): |
| parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, |
| description=description) |
| parser.add_argument('url', metavar='URL', |
| help='Points to the Jenkins log to parse') |
| parser.add_argument('--no-git', dest='useGit', action='store_false', default=True, |
| help='Do not run "git" at all') |
| parser.add_argument('--iters', dest='testIters', type=int, default=defaultIters, metavar='N', |
| help='Number of iterations per test suite (default: %d)' % defaultIters) |
| return parser.parse_args() |
| |
| def runOutput(cmd): |
| print('[repro] %s' % cmd) |
| try: |
| return subprocess.check_output(cmd.split(' '), universal_newlines=True).strip() |
| except CalledProcessError as e: |
| raise RuntimeError("ERROR: Cmd '%s' failed with exit code %d and the following output:\n%s" |
| % (cmd, e.returncode, e.output)) |
| |
| # Remembers non-zero exit code in lastFailureCode unless rememberFailure==False |
| def run(cmd, rememberFailure=True): |
| global lastFailureCode |
| print('[repro] %s' % cmd) |
| code = os.system(cmd) |
| if 0 != code and rememberFailure: |
| print('\n[repro] Setting last failure code to %d\n' % code) |
| lastFailureCode = code |
| return code |
| |
| def fetchAndParseJenkinsLog(url, numRetries): |
| global revisionFromLog |
| global branchFromLog |
| global antOptions |
| revisionFromLog = None |
| antOptions = '' |
| tests = {} |
| print('[repro] Jenkins log URL: %s\n' % url) |
| try: |
| # HTTPS fails at certificate validation, see LUCENE-9412, PEP-476 |
| context = ssl._create_unverified_context() |
| with urllib.request.urlopen(url, context=context) as consoleText: |
| for rawLine in consoleText: |
| line = rawLine.decode(encoding) |
| match = reGitRev.match(line) |
| if match is not None: |
| revisionFromLog = match.group(1) |
| branchFromLog = match.group(2) |
| print('[repro] Revision: %s\n' % revisionFromLog) |
| else: |
| match = reReproLine.search(line) |
| if match is not None: |
| print('[repro] Repro line: %s\n' % match.group(1)) |
| testcase = match.group(2) |
| reproLineWithoutMethod = match.group(3).strip() |
| tests[testcase] = reproLineWithoutMethod |
| else: |
| match = reAntInvocation.search(line) |
| if match is not None: |
| antOptions = ' '.join(reAntSysprops.findall(line)) |
| if len(antOptions) > 0: |
| print('[repro] Ant options: %s' % antOptions) |
| except urllib.error.URLError as e: |
| raise RuntimeError('ERROR: fetching %s : %s' % (url, e)) |
| except http.client.IncompleteRead as e: |
| if numRetries > 0: |
| print('[repro] Encountered IncompleteRead exception, pausing and then retrying...') |
| time.sleep(2) # pause for 2 seconds |
| return fetchAndParseJenkinsLog(url, numRetries - 1) |
| else: |
| print('[repro] Encountered IncompleteRead exception, aborting after too many retries.') |
| raise RuntimeError('ERROR: fetching %s : %s' % (url, e)) |
| |
| if revisionFromLog == None: |
| if reJenkinsURLWithoutConsoleText.match(url): |
| print('[repro] Not a Jenkins log. Appending "/consoleText" and retrying ...\n') |
| return fetchAndParseJenkinsLog(url + '/consoleText', numRetries) |
| else: |
| raise RuntimeError('ERROR: %s does not appear to be a Jenkins log.' % url) |
| if 0 == len(tests): |
| print('[repro] No "reproduce with" lines found; exiting.') |
| sys.exit(0) |
| return tests |
| |
| def prepareWorkspace(useGit, gitRef): |
| global gitCheckoutSucceeded |
| if useGit: |
| code = run('git fetch') |
| if 0 != code: |
| raise RuntimeError('ERROR: "git fetch" failed. See above.') |
| checkoutCmd = 'git checkout %s' % gitRef |
| code = run(checkoutCmd) |
| if 0 != code: |
| addWantedBranchCmd = "git remote set-branches --add origin %s" % gitRef |
| checkoutBranchCmd = 'git checkout -t -b %s origin/%s' % (gitRef, gitRef) # Checkout remote branch as new local branch |
| print('"%s" failed. Trying "%s" and "%s".' % (checkoutCmd, addWantedBranchCmd, checkoutBranchCmd)) |
| code = run(addWantedBranchCmd) |
| if 0 != code: |
| raise RuntimeError('ERROR: "%s" failed. See above.' % addWantedBranchCmd) |
| code = run(checkoutBranchCmd) |
| if 0 != code: |
| raise RuntimeError('ERROR: "%s" failed. See above.' % checkoutBranchCmd) |
| gitCheckoutSucceeded = True |
| run('git merge --ff-only', rememberFailure=False) # Ignore failure on non-branch ref |
| |
| code = run('ant clean') |
| if 0 != code: |
| raise RuntimeError('ERROR: "ant clean" failed. See above.') |
| |
| def groupTestsByModule(tests): |
| modules = {} |
| for (dir, _, files) in os.walk('.'): |
| for file in files: |
| match = reJavaFile.search(file) |
| if match is not None: |
| test = match.group(1) |
| if test in tests: |
| match = reModule.match(dir) |
| module = match.group(1) |
| if module not in modules: |
| modules[module] = set() |
| modules[module].add(test) |
| print('[repro] Test suites by module:') |
| for module in modules: |
| print('[repro] %s' % module) |
| for test in modules[module]: |
| print('[repro] %s' % test) |
| return modules |
| |
| def runTests(testIters, modules, tests): |
| cwd = os.getcwd() |
| testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s' |
| for module in modules: |
| moduleTests = list(modules[module]) |
| testList = '|'.join(map(lambda t: '*.%s' % t, moduleTests)) |
| numTests = len(moduleTests) |
| params = tests[moduleTests[0]] # Assumption: all tests in this module have the same cmdline params |
| os.chdir(module) |
| code = run('ant compile-test') |
| try: |
| if 0 != code: |
| raise RuntimeError("ERROR: Compile failed in %s/ with code %d. See above." % (module, code)) |
| run(testCmdline % (testIters, testIters * numTests, testList, antOptions, params)) |
| finally: |
| os.chdir(cwd) |
| |
| def printAndMoveReports(testIters, newSubDir, location): |
| failures = {} |
| for start in ('lucene/build', 'solr/build'): |
| for (dir, _, files) in os.walk(start): |
| for file in files: |
| testOutputFileMatch = reTestOutputFile.search(file) |
| if testOutputFileMatch is not None: |
| testcase = testOutputFileMatch.group(1) |
| if testcase not in failures: |
| failures[testcase] = 0 |
| filePath = os.path.join(dir, file) |
| with open(filePath, encoding='UTF-8') as testOutputFile: |
| for line in testOutputFile: |
| errorFailureMatch = reErrorFailure.search(line) |
| if errorFailureMatch is not None: |
| failures[testcase] += 1 |
| break |
| # have to play nice with 'ant clean'... |
| newDirPath = os.path.join('repro-reports', newSubDir, dir) |
| os.makedirs(newDirPath, exist_ok=True) |
| os.rename(filePath, os.path.join(newDirPath, file)) |
| print("[repro] Failures%s:" % location) |
| for testcase in sorted(failures, key=lambda t: (failures[t],t)): # sort by failure count, then by testcase |
| print("[repro] %d/%d failed: %s" % (failures[testcase], testIters, testcase)) |
| return failures |
| |
| def getLocalGitBranch(): |
| origGitBranch = runOutput('git rev-parse --abbrev-ref HEAD') |
| if origGitBranch == 'HEAD': # In detached HEAD state |
| origGitBranch = runOutput('git rev-parse HEAD') # Use the SHA when not on a branch |
| print('[repro] Initial local git branch/revision: %s' % origGitBranch) |
| return origGitBranch |
| |
| def main(): |
| config = readConfig() |
| tests = fetchAndParseJenkinsLog(config.url, numRetries = 2) |
| if config.useGit: |
| localGitBranch = getLocalGitBranch() |
| |
| try: |
| # have to play nice with ant clean, so printAndMoveReports will move all the junit XML files here... |
| print('[repro] JUnit rest result XML files will be moved to: ./repro-reports') |
| if os.path.isdir('repro-reports'): |
| print('[repro] Deleting old ./repro-reports'); |
| shutil.rmtree('repro-reports') |
| prepareWorkspace(config.useGit, revisionFromLog) |
| modules = groupTestsByModule(tests) |
| runTests(config.testIters, modules, tests) |
| failures = printAndMoveReports(config.testIters, 'orig', |
| ' w/original seeds' + (' at %s' % revisionFromLog if config.useGit else '')) |
| |
| |
| if config.useGit: |
| # Retest 100% failures at the tip of the branch |
| oldTests = tests |
| tests = {} |
| for fullClass in failures: |
| testcase = fullClass[(fullClass.rindex('.') + 1):] |
| if failures[fullClass] == config.testIters: |
| tests[testcase] = oldTests[testcase] |
| if len(tests) > 0: |
| print('\n[repro] Re-testing 100%% failures at the tip of %s' % branchFromLog) |
| prepareWorkspace(True, branchFromLog) |
| modules = groupTestsByModule(tests) |
| runTests(config.testIters, modules, tests) |
| failures = printAndMoveReports(config.testIters, 'branch-tip', |
| ' original seeds at the tip of %s' % branchFromLog) |
| |
| # Retest 100% tip-of-branch failures without a seed |
| oldTests = tests |
| tests = {} |
| for fullClass in failures: |
| testcase = fullClass[(fullClass.rindex('.') + 1):] |
| if failures[fullClass] == config.testIters: |
| tests[testcase] = re.sub(reTestsSeed, '', oldTests[testcase]) |
| if len(tests) > 0: |
| print('\n[repro] Re-testing 100%% failures at the tip of %s without a seed' % branchFromLog) |
| prepareWorkspace(False, branchFromLog) |
| modules = groupTestsByModule(tests) |
| runTests(config.testIters, modules, tests) |
| printAndMoveReports(config.testIters, 'branch-tip-no-seed', |
| ' at the tip of %s without a seed' % branchFromLog) |
| except Exception as e: |
| print('[repro] %s' % traceback.format_exc()) |
| sys.exit(1) |
| finally: |
| if config.useGit and gitCheckoutSucceeded: |
| run('git checkout %s' % localGitBranch, rememberFailure=False) # Restore original git branch/sha |
| |
| print('[repro] Exiting with code %d' % lastFailureCode) |
| sys.exit(lastFailureCode) |
| |
| if __name__ == '__main__': |
| try: |
| main() |
| except KeyboardInterrupt: |
| print('[repro] Keyboard interrupt...exiting') |
