| #!/usr/bin/env python |
| |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| ## |
| # Jenkins/Travis tools script, for monitoring and analyzing jobs from CI. |
| # |
| # Example use to monitor a travis build, job number N: |
| # > citool monitor N |
| # |
| # To monitor same job until completion: |
| # > citool monitor -p N |
| # |
| # To save job output to a file: |
| # > citool -o monitor N |
| # |
| # Example use to monitor a jenkins build B with job number N: |
| # > citool -u https://jenkins.host:port -b B monitor N |
| ## |
| |
| import sys |
| import time |
| import traceback |
| import argparse |
| import httplib |
| import re |
| import threading |
| import json |
| from xml.dom import minidom |
| from subprocess import Popen, PIPE, STDOUT |
| from urlparse import urlparse |
| |
| def main(): |
| exitCode = 0 |
| try: |
| args = parseArgs() |
| exitCode = { |
| 'monitor' : monitor, |
| 'cat': cat |
| }[args.cmd](args) |
| except Exception as e: |
| print('Exception: ', e) |
| if args.verbose: |
| traceback.print_exc() |
| exitCode = 1 |
| sys.exit(exitCode) |
| |
| def parseArgs(): |
| parser = argparse.ArgumentParser(description='tool for analyzing logs from CI') |
| subparsers = parser.add_subparsers(title='available commands', dest='cmd') |
| |
| parser.add_argument('job', help='job number') |
| parser.add_argument('-b', '--build', help='build name', default='travis') |
| parser.add_argument('-v', '--verbose', help='verbose output', action='store_true') |
| parser.add_argument('-i', '--input-file', help='read logs from file rather than CI', action='store_true', dest='ifile') |
| parser.add_argument('-o', '--output-file', help='store intermediate buffer to a file (e.g., jenkins console or component logs)', action='store_true', dest='ofile') |
| parser.add_argument('-u', '--url', help='URL for CI build job (default is Travis CI)', default='https://api.travis-ci.org') |
| |
| subparser = subparsers.add_parser('monitor', help='report passing or failing tests (only failing tests by default)') |
| subparser.add_argument('-a', '--all', help='show all tests suites, passing and failing', action='store_true') |
| subparser.add_argument('-p', '--poll', help='repeat monitor every 10 seconds', action='store_true') |
| |
| subparser = subparsers.add_parser('cat', help='concatenate logs from build (limited to Jenkins)') |
| subparser.add_argument('artifactPath', help='path to artifacts store') |
| subparser.add_argument('-g', '--grep', help='run grep against logs using provided value') |
| subparser.add_argument('-s', '--sort', help='sort logs by timestamp', action='store_true') |
| subparser.add_argument('-n', '--invokers', help='number of invokers', type=int, default=3) |
| subparser.add_argument('-c', '--controllers', help='number of controllers', type=int, default=1) |
| |
| return parser.parse_args() |
| |
| def request(method, urlString, body = "", headers = {}, auth = None, verbose = False): |
| url = urlparse(urlString) |
| if url.scheme == 'http': |
| conn = httplib.HTTPConnection(url.netloc) |
| else: |
| conn = httplib.HTTPSConnection(url.netloc) |
| |
| if verbose: |
| print("%s %s" % (method, urlString)) |
| |
| conn.request(method.upper(), urlString, body, headers = headers) |
| res = conn.getresponse() |
| |
| if verbose: |
| print('Got response with code %s' % res.status) |
| return res |
| |
| def shell(cmd, data = None, verbose = False): |
| start = time.time() |
| |
| if verbose: |
| print('%s%s' % (cmd, ' <stdin>' if (data) else '')) |
| |
| p = Popen(cmd, shell = True, stdout = PIPE, stderr = STDOUT, stdin = PIPE) |
| out, err = p.communicate(input = data) |
| |
| p.wait() |
| # stdout/stderr may be either text or bytes, depending on Python |
| # version. In the latter case, decode to text. |
| if isinstance(out, bytes): |
| out = out.decode('utf-8') |
| if isinstance(err, bytes): |
| err = err.decode('utf-8') |
| |
| end = time.time() |
| delta = end - start |
| return (delta, out, err) |
| |
| def getTravisHeaders(): |
| return {'User-Agent': 'wsk citool/0.0.1', |
| 'Accept': 'application/vnd.travis-ci.2+json' |
| } |
| |
| def getJobUrl(args): |
| if args.build.lower() == 'travis': |
| # Get build information |
| buildUrl = '%s/builds/%s' % (args.url, args.job) |
| buildRes = request('get', buildUrl, headers = getTravisHeaders(), verbose = args.verbose) |
| body = validateResponse(buildRes) |
| try: |
| body = json.loads(body) |
| job = body['build']['job_ids'][-1] |
| except Exception: |
| print('expected response to contain build and job-ids properties in %s' % body) |
| exit(-1) |
| url = '%s/jobs/%s' % (args.url, job) |
| else: # assume jenkins |
| url = '%s/job/%s/%s' % (args.url, args.build, args.job) |
| return url |
| |
| def monitor(args): |
| def poll(): |
| (ex, finished) = monitorOnce(args) |
| if not finished: |
| threading.Timer(10.0, poll).start() |
| |
| if args.poll: |
| poll() |
| else: |
| (ex, finished) = monitorOnce(args) |
| return ex |
| |
| def monitorOnce(args): |
| if args.ifile: |
| file = open('%s' % args.job, 'r') |
| body = file.read() |
| file.close() |
| else: |
| if args.build.lower() == 'travis': |
| url = '%s/log.txt' % getJobUrl(args) |
| res = request('get', url, verbose = args.verbose) |
| if res.status == httplib.TEMPORARY_REDIRECT: |
| url = res.getheader('location') |
| res = request('get', url, headers = getTravisHeaders(), verbose = args.verbose) |
| else: # assume jenkins |
| url = '%s/logText/progressiveHtml' % getJobUrl(args) |
| res = request('get', url, verbose = args.verbose) |
| |
| body = validateResponse(res) |
| |
| if args.ofile: |
| file = open('%s-console.log' % args.job, 'w') |
| file.write(body) |
| file.close() |
| if args.ifile or res.status == httplib.OK: |
| grepForFailingTests(args, body) |
| return reportBuildStatus(args, body) |
| elif args.ofile is False: |
| print(body) |
| return res.status |
| |
| def validateResponse(res): |
| body = res.read() |
| |
| if res.status != httplib.OK: |
| if body.startswith('<'): |
| dom = minidom.parseString(body) |
| print(dom.toprettyxml()), |
| else: |
| print(body) |
| exit(res.status) |
| elif not body: |
| print('build log is empty') |
| exit(-1) |
| else: |
| return body |
| |
| def grepForFailingTests(args, body): |
| cmd = 'grep :tests:test' |
| # check that tests ran |
| (time, output, error) = shell(cmd, body, args.verbose) |
| if output == '': |
| print('no tests detected') |
| # no tests: either build failure or task not yet reached, skip further check |
| else: |
| cmd = 'grep -E "^\w+\.*.*[>|>] \w*.* FAILED%s"' % ("|PASSED" if args.all else "") |
| (time, output, error) = shell(cmd, body, args.verbose) |
| if output == '': |
| print('all tests passing') |
| else: |
| print(output.replace('>', '>')), |
| |
| def reportBuildStatus(args, body): |
| lines = body.rstrip('\n').rsplit('\n', 1) |
| if len(lines) == 2: |
| output = lines[1] |
| output = re.sub('<[^<]+?>', '', output).strip() |
| |
| if output and ('Finished: ' in output or output.startswith('Done.') or ('exceeded' in output and 'terminated' in output)): |
| print(output) |
| return (0, True) |
| else: |
| print('Build: ONGOING') |
| if output: |
| print(output) |
| return (0, False) |
| |
| def cat(args): |
| def getComponentList(components): |
| list = [] |
| for k,v in components.items(): |
| if v > 1: |
| for i in range(v): |
| list.append('%s%d' % (k, i)) |
| else: |
| list.append(k) |
| return list |
| |
| def getComponentLogs(component): |
| url = '%s/artifact/%s/%s/%s_logs.log' % (getJobUrl(args), args.artifactPath, component, component) |
| res = request('get', url, verbose = args.verbose) |
| body = res.read() |
| if res.status == httplib.OK: |
| return body |
| else: |
| return '' |
| |
| def unzip(iterable): |
| return zip(*iterable) |
| |
| def extractDate(line): |
| matches = re.search(r'\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}T[0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}.\d{3}Z', line) |
| if matches is not None: |
| date = matches.group(0) |
| return date |
| else: |
| return None |
| |
| if args.ifile: |
| file = open('%s-build.log' % args.job, 'r') |
| joined = file.read() |
| file.close() |
| elif args.build.lower == 'travis': |
| print('feature not yet supported for Travis builds') |
| return 2 |
| else: |
| components = { |
| 'controller': args.controllers, |
| 'invoker': args.invokers |
| } |
| logs = map(getComponentLogs, getComponentList(components)) |
| joined = ''.join(logs) |
| if args.ofile: |
| file = open('%s-build.log' % args.job, 'w') |
| file.write(joined) |
| file.close() |
| |
| if args.grep is not None: |
| cmd = 'grep "%s"' % args.grep |
| (time, output, error) = shell(cmd, joined, args.verbose) |
| output = output.strip() |
| if args.sort: |
| parts = output.split('\n') |
| filter = [p for p in parts if p != ''] |
| date = map(extractDate, filter) |
| keyed = zip(date, parts) |
| sort = sorted(keyed, key=lambda t: t[1]) |
| msgs = unzip(sort)[1] |
| print('\n'.join(msgs)) |
| return 0 |
| else: |
| print(output) |
| return 0 |
| elif args.ofile is False: |
| print(joined) |
| return 0 |
| |
| if __name__ == '__main__': |
| main() |
