| #!/usr/bin/env python |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| |
| from __future__ import print_function |
| |
| import datetime |
| import glob |
| import json |
| import os |
| import tarfile |
| import time |
| |
| import requests |
| |
| COUCH_URL = "https://couchdb-vm2.apache.org/ci_errorlogs" |
| TARFILE = "couchlog.tar.gz" |
| |
| def _tojson(req): |
| """Support requests v0.x as well as 1.x+""" |
| if requests.__version__[0] == '0': |
| return json.loads(req.content) |
| return req.json() |
| |
| def collect_logfiles(): |
| """ Find and tarball all logfiles """ |
| tb = tarfile.open(name=TARFILE, mode='w:gz') |
| # EUnit |
| for log in glob.glob('src/*/.eunit/couch.log'): |
| tb.add(log) |
| # JS harness |
| for log in glob.glob('dev/logs/node1.log'): |
| tb.add(log) |
| # couchjs OS process IO logs |
| for log in glob.glob('/tmp/couchjslogs/*'): |
| tb.add(log) |
| tb.close() |
| |
| def build_ci_doc(): |
| """ Build a metadata document with relevant detail from CI env """ |
| doc = {} |
| if 'TRAVIS' in os.environ: |
| doc['builder'] = 'travis' |
| doc['build_id'] = os.environ['TRAVIS_JOB_ID'] |
| doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE'] |
| doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \ |
| os.environ['TRAVIS_JOB_ID'] |
| doc['branch'] = os.environ['TRAVIS_BRANCH'] |
| doc['commit'] = os.environ['TRAVIS_COMMIT'] |
| doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG'] |
| elif 'JENKINS_URL' in os.environ: |
| doc['builder'] = 'jenkins' |
| doc['build_id'] = os.environ['BUILD_NUMBER'] |
| doc['url'] = os.environ['BUILD_URL'] |
| doc['branch'] = os.environ['BRANCH_NAME'] |
| doc['repo'] = 'https://github.com/apache/couchdb' |
| else: |
| doc['builder'] = 'manual' |
| # TODO: shell out to get correct repo, commit, branch info? |
| doc['repo'] = 'https://github.com/apache/couchdb' |
| doc['build_id'] = str(time.time()) |
| |
| # shorten doc id |
| repo = doc['repo'].split('/')[-1] |
| repo = repo.replace('.git', '') |
| |
| doc['_id'] = doc['builder'] + '-' + repo + '-' + \ |
| doc['build_id'] + \ |
| '-' + datetime.datetime.utcnow().isoformat() |
| |
| return doc |
| |
| def upload_logs(): |
| try: |
| lp = os.environ['COUCHAUTH'].split(':') |
| except KeyError as e: |
| print ("ERROR: COUCHAUTH credentials unavailable! " |
| "Unable to upload logfiles.") |
| exit(1) |
| |
| creds = (lp[0], lp[1]) |
| doc = build_ci_doc() |
| req = requests.post(COUCH_URL, |
| data=json.dumps(doc), |
| auth=creds, |
| headers={'Content-type': 'application/json'}) |
| req.raise_for_status() |
| req = _tojson(req) |
| with open(TARFILE, 'rb') as f: |
| # ancient versions of requests break if data is iterable |
| fdata = f.read() |
| req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE, |
| headers={'Content-type': 'application/x-gtar'}, |
| auth=creds, |
| params={'rev': req['rev']}, |
| data=fdata) |
| req2.raise_for_status() |
| return req2 |
| |
| |
| def main(): |
| """ Find latest logfile and upload to Couch logfile db. """ |
| print ("Uploading logfiles...") |
| collect_logfiles() |
| req = upload_logs() |
| print (req.url.split('?')[0]) |
| print (req.content) |
| print ("Upload complete!") |
| |
| if __name__ == '__main__': |
| main() |