dev/airflow-jira

#!/usr/bin/env python

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Utility for creating well-formed pull request merges and pushing them to
# Apache.
#
#   usage: ./airflow-jira    (see config env vars below)
#
# This utility assumes you already have a local Airflow git folder and that you
# have added remotes corresponding to both (i) the github apache Airflow
# mirror and (ii) the apache git repo.

# This tool is based on the Spark merge_spark_pr script:
# https://github.com/apache/spark/blob/master/dev/merge_spark_pr.py

from __future__ import print_function

import jira
import re
import sys

TMP_CREDENTIALS = {}
PROJECT = "AIRFLOW"

# Python 3 compatibility
if sys.version_info[0] == 3:
    raw_input = input

try:
    import click
except ImportError:
    print("Could not find the click library. Run 'sudo pip install click' to install.")
    sys.exit(-1)

try:
    import git
except ImportError:
    print("Could not import git. Run 'sudo pip install gitpython' to install")
    sys.exit(-1)

JIRA_BASE = "https://issues.apache.org/jira/browse"
JIRA_API_BASE = "https://issues.apache.org/jira"

GIT_COMMIT_FIELDS = ['id', 'author_name', 'author_email', 'date', 'subject', 'body']
GIT_LOG_FORMAT = '%x1f'.join(['%h', '%an', '%ae', '%ad', '%s', '%b']) + '%x1e'

STATUS_COLUR_MAP = {
    'Resolved': 'green',
    'Open': 'red',
    'Closed': 'yellow'
}


def get_jiras_for_version(version):
    asf_jira = jira.client.JIRA({'server': JIRA_API_BASE})

    start_at = 0
    page_size = 50
    while True:
        results = asf_jira.search_issues(
            'PROJECT={} and fixVersion={} order by updated desc'.format(PROJECT, version),
            maxResults=page_size,
            startAt=start_at,
        )

        for r in results:
            yield r

        if len(results) < page_size:
            break

        start_at += page_size


issue_re = re.compile(r".*? (AIRFLOW-[0-9]{1,6}) (?: \]|\s|: )", flags=re.X)
pr_re = re.compile(r"(.*)Closes (#[0-9]{1,6})", flags=re.DOTALL)
pr_title_re = re.compile(r".*\((#[0-9]{1,6})\)$")


def get_merged_issues(repo, version, previous_version=None):
    log_args = ['--format={}'.format(GIT_LOG_FORMAT)]
    if previous_version:
        log_args.append(previous_version + "..")
    log = repo.git.log(*log_args)

    log = log.strip('\n\x1e').split("\x1e")
    log = [row.strip().split("\x1f") for row in log]
    log = [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]

    merges = {}
    for log_item in log:
        issue_id = None

        issue_ids = issue_re.findall(log_item['subject'])

        match = pr_title_re.match(log_item['subject'])
        if match:
            log_item['pull_request'] = match.group(1)
        elif 'body' in log_item:
            match = pr_re.match(log_item['body'])
            if match:
                log_item['pull_request'] = match.group(2)
            else:
                log_item['pull_request'] = '#na'
        else:
            log_item['pull_request'] = '#na'

        for issue_id in issue_ids:
            merges[issue_id] = log_item

    return merges


def get_commits_from_master(repo, issue):
    log = repo.git.log(
        '--format=%h%x1f%s',
        '-1',
        '--grep',
        r'.*{issue}\(\]\|:\|\s\)'.format(issue=issue.key),
        'origin/master')
    if not log:
        return None
    commit, subject = log.split('\x1f')

    merge = {'id': commit}
    match = pr_title_re.match(subject)

    if match:
        merge['pull_request'] = match.group(1)
    else:
        merge['pull_request'] = '-'

    return merge


@click.group()
def cli():
    r"""
    This tool should be used by Airflow Release Manager to verify what Jira's
     were merged in the current working branch.

        airflow-jira compare <target_version>
    """


@cli.command(short_help='Compare a jira target version against git merges')
@click.argument('target_version', default=None)
@click.argument('previous_version', default="")
@click.option('--unmerged', 'unmerged_only', help="Show unmerged issues only", is_flag=True)
def compare(target_version, previous_version=None, unmerged_only=False):
    repo = git.Repo(".", search_parent_directories=True)
    merges = get_merged_issues(repo, target_version, previous_version)
    issues = get_jiras_for_version(target_version)

    # :<18 says left align, pad to 18
    # :<50.50 truncates after 50 chars
    # !s forces as string - some of the Jira objects have a string method, but
    #    Py3 doesn't call by default
    formatstr = "{id:<18}|{typ!s:<12}||{priority!s:<10}||{status!s}|" \
                "{description:<83.83}|{merged:<6}|{pr:<6}|{commit:<7}"

    print(formatstr.format(
        id="ISSUE ID",
        typ="TYPE",
        priority="PRIORITY",
        status="STATUS".ljust(10),
        description="DESCRIPTION",
        merged="MERGED",
        pr="PR",
        commit="COMMIT"))

    for issue in issues:
        is_merged = issue.key in merges

        if unmerged_only and is_merged:
            continue

        # Put colour on the status field. Since it will have non-printable
        # characters we can't us string format to limit the length
        status = issue.fields.status.name
        if status in STATUS_COLUR_MAP:
            status = click.style(status[:10].ljust(10), STATUS_COLUR_MAP[status])
        else:
            status = status[:10].ljust(10)

        merge = merges.get(issue.key)
        if not merge and issue.fields.status.name in {'Resolved', 'Closed'}:
            merge = get_commits_from_master(repo, issue)

        print(formatstr.format(
            id=issue.key,
            typ=issue.fields.issuetype,
            priority=issue.fields.priority,
            status=status,
            description=issue.fields.summary,
            merged=is_merged,
            pr=merge['pull_request'] if merge else "-",
            commit=merge['id'] if merge else "-"))


if __name__ == "__main__":
    import doctest
    (failure_count, test_count) = doctest.testmod()
    if failure_count:
        exit(-1)
    try:
        cli()
    except Exception:
        raise