build-aux/logfile-uploader.py

#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from __future__ import print_function

import datetime
import glob
import json
import os
import tarfile
import time

import requests

COUCH_URL = "https://couchdb-vm2.apache.org/ci_errorlogs"
TARFILE = "couchlog.tar.gz"

def _tojson(req):
    """Support requests v0.x as well as 1.x+"""
    if requests.__version__[0] == '0':
        return json.loads(req.content)
    return req.json()

def collect_logfiles():
    """ Find and tarball all logfiles """
    tb = tarfile.open(name=TARFILE, mode='w:gz')
    # EUnit
    for log in glob.glob('src/*/.eunit/couch.log'):
        tb.add(log)
    # JS harness
    for log in glob.glob('dev/logs/node1.log'):
        tb.add(log)
    # couchjs OS process IO logs
    for log in glob.glob('/tmp/couchjslogs/*'):
        tb.add(log)
    tb.close()

def build_ci_doc():
    """ Build a metadata document with relevant detail from CI env """
    doc = {}
    if 'TRAVIS' in os.environ:
        doc['builder'] = 'travis'
        doc['build_id'] = os.environ['TRAVIS_JOB_ID']
        doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE']
        doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \
            os.environ['TRAVIS_JOB_ID']
        doc['branch'] = os.environ['TRAVIS_BRANCH']
        doc['commit'] = os.environ['TRAVIS_COMMIT']
        doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG']
    elif 'JENKINS_URL' in os.environ:
        doc['builder'] = 'jenkins'
        doc['build_id'] = os.environ['BUILD_NUMBER']
        doc['url'] = os.environ['BUILD_URL']
        doc['branch'] = os.environ['BRANCH_NAME']
        doc['repo'] = 'https://github.com/apache/couchdb'
    else:
        doc['builder'] = 'manual'
        # TODO: shell out to get correct repo, commit, branch info?
        doc['repo'] = 'https://github.com/apache/couchdb'
        doc['build_id'] = str(time.time())

    # shorten doc id
    repo = doc['repo'].split('/')[-1]
    repo = repo.replace('.git', '')

    doc['_id'] = doc['builder'] + '-' + repo + '-' + \
        doc['build_id'] + \
        '-' + datetime.datetime.utcnow().isoformat()

    return doc

def upload_logs():
    try:
        lp = os.environ['COUCHAUTH'].split(':')
    except KeyError as e:
        print ("ERROR: COUCHAUTH credentials unavailable! "
            "Unable to upload logfiles.")
        exit(1)

    creds = (lp[0], lp[1])
    doc = build_ci_doc()
    req = requests.post(COUCH_URL,
        data=json.dumps(doc),
        auth=creds,
        headers={'Content-type': 'application/json'})
    req.raise_for_status()
    req = _tojson(req)
    with open(TARFILE, 'rb') as f:
        # ancient versions of requests break if data is iterable
        fdata = f.read()
        req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE,
            headers={'Content-type': 'application/x-gtar'},
            auth=creds,
            params={'rev': req['rev']},
            data=fdata)
    req2.raise_for_status()
    return req2


def main():
    """ Find latest logfile and upload to Couch logfile db. """
    print ("Uploading logfiles...")
    collect_logfiles()
    req = upload_logs()
    print (req.url.split('?')[0])
    print (req.content)
    print ("Upload complete!")

if __name__ == '__main__':
    main()