| #!/usr/bin/env python3.4 |
| # -*- coding: utf-8 -*- |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| #the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import re |
| import hashlib |
| from dateutil import parser |
| import time |
| import requests |
| import plugins.utils.github |
| |
| title = "Scanner for GitHub Issues" |
| version = "0.1.0" |
| |
| def accepts(source): |
| """ Return true if this is a github repo """ |
| if source['type'] == 'github': |
| return True |
| if source['type'] == 'git' and re.match(r"https://(?:www\.)?github.com/", source['sourceURL']): |
| return True |
| return False |
| |
| def format_date(d, epoch=False): |
| if not d: |
| return |
| parsed = parser.parse(d) |
| |
| if epoch: |
| return time.mktime(parsed.timetuple()) |
| |
| return time.strftime("%Y/%m/%d %H:%M:%S", parsed.timetuple()) |
| |
| def make_hash(source, issue): |
| return hashlib.sha224(("%s-%s-%s" % (source['organisation'], |
| source['sourceID'], |
| str(issue['id']))).encode('ascii', |
| errors='replace')).hexdigest() |
| |
| def make_issue(source, issue, people): |
| |
| key = str(issue['number']) |
| dhash = make_hash(source, issue) |
| |
| closed_date = issue.get('closed_at', None) |
| |
| owner_email = people[issue['user']['login']]['email'] |
| |
| issue_closer = owner_email |
| if 'closed_by' in issue: |
| issue_closer = people[issue['closed_by']['login']] |
| # Is this an issue ro a pull request? |
| itype = "issue" |
| if 'pull_request' in issue: |
| itype = "pullrequest" |
| labels = [] |
| for l in issue.get('labels', []): |
| labels.append(l['name']) |
| return { |
| 'id': dhash, |
| 'key': key, |
| 'issuetype': itype, |
| 'organisation': source['organisation'], |
| 'sourceID': source['sourceID'], |
| 'url': issue['html_url'], |
| 'status': issue['state'], |
| 'labels': labels, |
| 'created': format_date(issue['created_at'], epoch=True), |
| 'closed': format_date(closed_date, epoch=True), |
| 'issueCloser': issue_closer, |
| 'createdDate': format_date(issue['created_at']), |
| 'closedDate': format_date(closed_date), |
| 'changeDate': format_date(closed_date |
| if closed_date |
| else issue['updated_at']), |
| 'assignee': owner_email, |
| 'issueCreator': owner_email, |
| 'comments': issue['comments'], |
| 'title': issue['title'] |
| } |
| |
| def make_person(source, issue, raw_person): |
| email = raw_person['email'] |
| if not email: |
| email = "%s@invalid.github.com" % issue['user']['login'] |
| |
| name = raw_person['name'] |
| if not name: |
| name = raw_person['login'] |
| |
| id = hashlib.sha1(("%s%s" % (source['organisation'], |
| email)).encode('ascii', |
| errors='replace')).hexdigest() |
| |
| return {'email': email, 'id': id, 'organisation': source['organisation'], |
| 'name': name} |
| |
| def status_changed(stored_issue, issue): |
| return stored_issue['status'] != issue['status'] |
| |
| def update_issue(KibbleBit, issue): |
| KibbleBit.append('issue', issue) |
| |
| def update_person(KibbleBit, person): |
| person['upsert'] = True |
| KibbleBit.append('person', person) |
| |
| |
| def scan(KibbleBit, source): |
| auth=None |
| people = {} |
| if 'creds' in source: |
| KibbleBit.pprint("Using auth for repo %s" % source['sourceURL']) |
| creds = source['creds'] |
| if creds and 'username' in creds: |
| auth = (creds['username'], creds['password']) |
| KibbleBit.pprint("Scanning for GitHub issues") |
| source['steps']['issues'] = { |
| 'time': time.time(), |
| 'status': 'Issue scan started at ' + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), |
| 'running': True, |
| 'good': True, |
| } |
| KibbleBit.updateSource(source) |
| try: |
| issues = plugins.utils.github.get_all(source, plugins.utils.github.issues, |
| params={'filter': 'all', 'state':'all'}, |
| auth=auth) |
| KibbleBit.pprint("Fetched %s issues for %s" %(str(len(issues)), source['sourceURL'])) |
| |
| for issue in issues: |
| |
| if not issue['user']['login'] in people: |
| person = make_person(source, issue, plugins.utils.github.user(issue['user']['url'], |
| auth=auth)) |
| people[issue['user']['login']] = person |
| update_person(KibbleBit, person) |
| |
| if 'closed_by' in issue and not issue['closed_by']['login'] in people: |
| closer = make_person(source, issue, plugins.utils.github.user(issue['closed_by']['url'], |
| auth=auth)) |
| people[issue['closed_by']['login']] = closer |
| update_person(KibbleBit, closer) |
| |
| doc = make_issue(source, issue, people) |
| dhash = doc['id'] |
| stored_change = None |
| if KibbleBit.exists('issue', dhash): |
| es_doc = KibbleBit.get('issue', dhash) |
| if not status_changed(es_doc, doc): |
| KibbleBit.pprint("change %s seen already and status unchanged. Skipping." % issue['id']) |
| continue |
| |
| update_issue(KibbleBit, doc) |
| |
| source['steps']['issues'] = { |
| 'time': time.time(), |
| 'status': 'Issue scan completed at ' + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), |
| 'running': False, |
| 'good': True, |
| } |
| KibbleBit.updateSource(source) |
| |
| except requests.HTTPError as e: |
| # we've likely hit our GH API quota for the hour, so we re-try |
| KibbleBit.pprint("HTTP Error, rate limit exceeded?") |
| source['steps']['issues'] = { |
| 'time': time.time(), |
| 'status': 'Issue scan failed at ' + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) + " - rate limited or no auth used??", |
| 'running': False, |
| 'good': False, |
| } |
| KibbleBit.updateSource(source) |