| #!/usr/bin/env python |
| ## |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| # Makes a patch for the current branch, creates/updates the review board request and uploads new |
| # patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming |
| # rules. If no jira is specified, patch will be named (branch name).patch and jira and review board |
| # are not updated. Review board id is retrieved from the remote link in the jira. |
| # Print help: submit-patch.py --h |
| import argparse |
| from builtins import input, str |
| import getpass |
| import git |
| import json |
| import logging |
| import os |
| import re |
| import requests |
| import subprocess |
| import sys |
| |
| parser = argparse.ArgumentParser( |
| epilog = "To avoid having to enter jira/review board username/password every time, setup an " |
| "encrypted ~/.apache-cred files as follows:\n" |
| "1) Create a file with following single " |
| "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", " |
| "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n" |
| "2) Encrypt it with openssl.\n" |
| "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n" |
| "3) Delete original file.\n" |
| "Now onwards, you'll need to enter this encryption key only once per run. If you " |
| "forget the key, simply regenerate ~/.apache-cred file again.", |
| formatter_class=argparse.RawTextHelpFormatter |
| ) |
| parser.add_argument("-b", "--branch", |
| help = "Branch to use for generating diff. If not specified, tracking branch " |
| "is used. If there is no tracking branch, error will be thrown.") |
| |
| # Arguments related to Jira. |
| parser.add_argument("-jid", "--jira-id", |
| help = "Jira id of the issue. If set, we deduce next patch version from " |
| "attachments in the jira and also upload the new patch. Script will " |
| "ask for jira username/password for authentication. If not set, " |
| "patch is named <branch>.patch.") |
| |
| # Arguments related to Review Board. |
| parser.add_argument("-srb", "--skip-review-board", |
| help = "Don't create/update the review board.", |
| default = False, action = "store_true") |
| parser.add_argument("--reviewers", |
| help = "Comma separated list of users to add as reviewers.") |
| |
| # Misc arguments |
| parser.add_argument("--patch-dir", default = "~/patches", |
| help = "Directory to store patch files. If it doesn't exist, it will be " |
| "created. Default: ~/patches") |
| parser.add_argument("--rb-repo", default = "hbase-git", |
| help = "Review board repository. Default: hbase-git") |
| args = parser.parse_args() |
| |
| # Setup logger |
| logging.basicConfig() |
| logger = logging.getLogger("submit-patch") |
| logger.setLevel(logging.INFO) |
| |
| |
| def log_fatal_and_exit(*arg): |
| logger.fatal(*arg) |
| sys.exit(1) |
| |
| |
| def assert_status_code(response, expected_status_code, description): |
| if response.status_code != expected_status_code: |
| log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..", |
| description, response.status_code, response.reason) |
| |
| |
| # Make repo instance to interact with git repo. |
| try: |
| repo = git.Repo(os.getcwd()) |
| git = repo.git |
| except git.exc.InvalidGitRepositoryError as e: |
| log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of " |
| "HBase's git repo.", e) |
| |
| logger.info(" Active branch: %s", repo.active_branch.name) |
| # Do not proceed if there are uncommitted changes. |
| if repo.is_dirty(): |
| log_fatal_and_exit(" Git status is dirty. Commit locally first.") |
| |
| |
| # Returns base branch for creating diff. |
| def get_base_branch(): |
| # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch. |
| if args.branch is not None: |
| base_branch = args.branch |
| # Check that given branch exists. |
| for ref in repo.refs: |
| if ref.name == base_branch: |
| return base_branch |
| log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch) |
| else: |
| # if --branch is not set, use tracking branch as base branch for computing diff. |
| # If there is no tracking branch, log error and quit. |
| tracking_branch = repo.active_branch.tracking_branch() |
| if tracking_branch is None: |
| log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base " |
| " branch for computing diff using --branch flag.") |
| logger.info(" Using tracking branch as base branch") |
| return tracking_branch.name |
| |
| |
| # Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is |
| # specified, patch is name (branch name).patch. |
| def get_patch_name(branch): |
| if args.jira_id is None: |
| return branch + ".patch" |
| |
| patch_name_prefix = args.jira_id.upper() + "." + branch |
| return get_patch_name_with_version(patch_name_prefix) |
| |
| |
| # Fetches list of attachments from the jira, deduces next version for the patch and returns final |
| # patch name. |
| def get_patch_name_with_version(patch_name_prefix): |
| # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637. |
| # Using crude way to get list of attachments. |
| url = "https://issues.apache.org/jira/browse/" + args.jira_id |
| logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url) |
| html = requests.get(url) |
| if html.status_code == 404: |
| log_fatal_and_exit(" Invalid jira id : %s", args.jira_id) |
| if html.status_code != 200: |
| log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code) |
| # Iterate over patch names starting from version 1 and return when name is not already used. |
| content = str(html.content, 'utf-8') |
| for i in range(1, 1000): |
| name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch" |
| if name not in content: |
| return name |
| |
| |
| # Validates that patch directory exists, if not, creates it. |
| def validate_patch_dir(patch_dir): |
| # Create patch_dir if it doesn't exist. |
| if not os.path.exists(patch_dir): |
| logger.warn(" Patch directory doesn't exist. Creating it.") |
| os.mkdir(patch_dir) |
| else: |
| # If patch_dir exists, make sure it's a directory. |
| if not os.path.isdir(patch_dir): |
| log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.", |
| patch_dir) |
| |
| |
| # Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if |
| # - base_branch has commits not in current branch |
| # - current branch is same as base branch |
| # - current branch is ahead of base_branch by more than 1 commits |
| def check_diff_between_branches(base_branch): |
| only_in_base_branch = list(repo.iter_commits("HEAD.." + base_branch)) |
| only_in_active_branch = list(repo.iter_commits(base_branch + "..HEAD")) |
| if len(only_in_base_branch) != 0: |
| log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase " |
| "and try again.", base_branch, len(only_in_base_branch)) |
| if len(only_in_active_branch) == 0: |
| log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch) |
| if len(only_in_active_branch) > 1: |
| log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single " |
| "commit and try again.", base_branch, len(only_in_active_branch)) |
| |
| |
| # If ~/.apache-creds is present, load credentials from it otherwise prompt user. |
| def get_credentials(): |
| creds = dict() |
| creds_filepath = os.path.expanduser("~/.apache-creds") |
| if os.path.exists(creds_filepath): |
| try: |
| logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials") |
| content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath, |
| shell=True) |
| except subprocess.CalledProcessError as e: |
| log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..") |
| creds = json.loads(content) |
| else: |
| creds['jira_username'] = input("Jira username:") |
| creds['jira_password'] = getpass.getpass("Jira password:") |
| if not args.skip_review_board: |
| creds['rb_username'] = input("Review Board username:") |
| creds['rb_password'] = getpass.getpass("Review Board password:") |
| return creds |
| |
| |
| def attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds): |
| # Upload patch to jira using REST API. |
| headers = {'X-Atlassian-Token': 'no-check'} |
| files = {'file': (patch_filename, open(patch_filepath, 'rb'), 'text/plain')} |
| jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password']) |
| attachment_url = issue_url + "/attachments" |
| r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth) |
| assert_status_code(r, 200, "uploading patch to jira") |
| |
| |
| def get_jira_summary(issue_url): |
| r = requests.get(issue_url + "?fields=summary") |
| assert_status_code(r, 200, "fetching jira summary") |
| return json.loads(r.content)["fields"]["summary"] |
| |
| |
| def get_review_board_id_if_present(issue_url, rb_link_title): |
| r = requests.get(issue_url + "/remotelink") |
| assert_status_code(r, 200, "fetching remote links") |
| links = json.loads(r.content) |
| for link in links: |
| if link["object"]["title"] == rb_link_title: |
| res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"]) |
| return res.group(1) |
| return None |
| |
| |
| base_branch = get_base_branch() |
| # Remove remote repo name from branch name if present. This assumes that we don't use '/' in |
| # actual branch names. |
| base_branch_without_remote = base_branch.split('/')[-1] |
| logger.info(" Base branch: %s", base_branch) |
| |
| check_diff_between_branches(base_branch) |
| |
| patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir)) |
| logger.info(" Patch directory: %s", patch_dir) |
| validate_patch_dir(patch_dir) |
| |
| patch_filename = get_patch_name(base_branch_without_remote) |
| logger.info(" Patch name: %s", patch_filename) |
| patch_filepath = os.path.join(patch_dir, patch_filename) |
| |
| diff = git.format_patch(base_branch, stdout = True) |
| with open(patch_filepath, "wb") as f: |
| f.write(diff.encode('utf8')) |
| |
| if args.jira_id is not None: |
| creds = get_credentials() |
| issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id |
| |
| attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds) |
| |
| if not args.skip_review_board: |
| rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password']) |
| |
| rb_link_title = "Review Board (" + base_branch_without_remote + ")" |
| rb_id = get_review_board_id_if_present(issue_url, rb_link_title) |
| |
| # If no review board link found, create new review request and add its link to jira. |
| if rb_id is None: |
| reviews_url = "https://reviews.apache.org/api/review-requests/" |
| data = {"repository" : "hbase-git"} |
| r = requests.post(reviews_url, data = data, auth = rb_auth) |
| assert_status_code(r, 201, "creating new review request") |
| review_request = json.loads(r.content)["review_request"] |
| absolute_url = review_request["absolute_url"] |
| logger.info(" Created new review request: %s", absolute_url) |
| |
| # Use jira summary as review's summary too. |
| summary = get_jira_summary(issue_url) |
| # Use commit message as description. |
| description = repo.head.commit.message |
| update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "hbase", |
| "target_people" : args.reviewers, "summary" : summary, |
| "description" : description } |
| draft_url = review_request["links"]["draft"]["href"] |
| r = requests.put(draft_url, data = update_draft_data, auth = rb_auth) |
| assert_status_code(r, 200, "updating review draft") |
| |
| draft_request = json.loads(r.content)["draft"] |
| diff_url = draft_request["links"]["draft_diffs"]["href"] |
| files = {'path' : (patch_filename, open(patch_filepath, 'rb'))} |
| r = requests.post(diff_url, files = files, auth = rb_auth) |
| assert_status_code(r, 201, "uploading diff to review draft") |
| |
| r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth) |
| assert_status_code(r, 200, "publishing review request") |
| |
| # Add link to review board in the jira. |
| remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}}) |
| jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password']) |
| r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth, |
| headers={'Content-Type':'application/json'}) |
| else: |
| logger.info(" Updating existing review board: https://reviews.apache.org/r/%s", rb_id) |
| draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/" |
| diff_url = draft_url + "diffs/" |
| files = {'path' : (patch_filename, open(patch_filepath, 'rb'))} |
| r = requests.post(diff_url, files = files, auth = rb_auth) |
| assert_status_code(r, 201, "uploading diff to review draft") |
| |
| r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth) |
| assert_status_code(r, 200, "publishing review request") |
