#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This script is used to build and test (verify) reviews that are posted
to ReviewBoard. The script is intended for use by automated "ReviewBots"
that are run on ASF infrastructure (or by anyone that wishes to donate
some compute power). For example, see 'support/jenkins/reviewbot.sh'.

The script performs the following sequence:
* A query grabs review IDs from Reviewboard.
* In reverse order (most recent first), the script determines if the
  review needs verification (if the review has been updated or changed
  since the last run through this script).
* For each review that needs verification:
  * The review is applied (via 'support/apply-reviews.py').
  * Mesos is built and unit tests are run.
  * The result is posted to ReviewBoard.
"""

import argparse
import atexit
import json
import os
import platform
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request

from datetime import datetime

from common import REVIEWBOARD_URL

REVIEW_SIZE = 1000000  # 1 MB in bytes.

# Parse arguments.
parser = argparse.ArgumentParser(
    description="Reviews that need verification from the Review Board")
parser.add_argument(
    "-u", "--user", type=str, required=True, help="Review Board user name")
parser.add_argument(
    "-p",
    "--password",
    type=str,
    required=True,
    help="Review Board user password")
parser.add_argument(
    "-r",
    "--reviews",
    type=int,
    required=False,
    default=-1,
    help="The number of reviews to fetch,"
    " that will need verification")

# Unless otherwise specified consider pending review requests to Mesos updated
# since 03/01/2014. Note that we need to explicitly set `max-results` query
# parameter because the default is 25 (max allowed is 200).
# TODO(vinod): Paginate through the API to get all pending review requests
# instead of just getting the first 200.
group = "mesos"
last_updated = "2014-03-01T00:00:00"
query_parameters = \
    "?to-groups=%s&status=pending&last-updated-from=%s&max-results=200" \
    % (group, last_updated)
parser.add_argument(
    "-q",
    "--query",
    type=str,
    required=False,
    help="Query parameters",
    default=query_parameters)

parser.add_argument(
    "-o",
    "--out-file",
    type=str,
    required=False,
    help="The out file with the reviews IDs that"
    " need verification")
parser.add_argument(
    "--skip-verify",
    action='store_true',
    required=False,
    help="Skip the verification and just write the review"
    " ids that need verification")

parameters = parser.parse_args()
USER = parameters.user
PASSWORD = parameters.password
NUM_REVIEWS = parameters.reviews
QUERY_PARAMS = parameters.query
OUT_FILE = parameters.out_file
SKIP_VERIFY = parameters.skip_verify


class ReviewError(Exception):
    """Exception returned by post_review()."""


def parse_time(timestamp):
    """Parse time in ReviewBoard date format."""
    return datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ")


def shell(command, working_dir=None):
    """Run a shell command."""
    out = subprocess.check_output(
        command, stderr=subprocess.STDOUT, cwd=working_dir, shell=True)
    return out.decode(sys.stdout.encoding)


SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
HEAD = shell("git rev-parse HEAD", SCRIPT_PATH)


def api(url, data=None):
    """Call the ReviewBoard API."""
    try:
        auth_handler = urllib.request.HTTPBasicAuthHandler()
        auth_handler.add_password(
            realm="Web API",
            uri="reviews.apache.org",
            user=USER,
            passwd=PASSWORD)

        opener = urllib.request.build_opener(auth_handler)
        urllib.request.install_opener(opener)

        if isinstance(data, str):
            data = str.encode(data)
        f = urllib.request.urlopen(url, data=data)
        return json.loads(f.read().decode("utf-8"))
    except urllib.error.HTTPError as err:
        print("Error handling URL %s: %s (%s)" % (url, err.reason, err.read()))
        exit(1)
    except urllib.error.URLError as err:
        print("Error handling URL %s: %s" % (url, err.reason))
        exit(1)


def apply_review(review_id):
    """Apply a review using the script apply-reviews.py."""
    print("Applying review %s" % review_id)
    shell(
        "cd .. && %s support/apply-reviews.py -n -r %s" %
        (sys.executable, review_id), SCRIPT_PATH)


def apply_reviews(review_request, reviews):
    """Apply multiple reviews at once."""
    # If there are no reviewers specified throw an error.
    if not review_request["target_people"]:
        raise ReviewError("No reviewers specified. Please find a reviewer by"
                          " asking on JIRA or the mailing list.")

    # If there is a circular dependency throw an error.`
    if review_request["id"] in reviews:
        raise ReviewError(
            "Circular dependency detected for review %s."
            "Please fix the 'depends_on' field." % review_request["id"])

    reviews.append(review_request["id"])

    # First recursively apply the dependent reviews.
    for review in review_request["depends_on"]:
        review_url = review["href"]
        print("Dependent review: %s " % review_url)
        apply_reviews(api(review_url)["review_request"], reviews)

    # Now apply this review if not yet submitted.
    if review_request["status"] != "submitted":
        # If the patch does not apply we translate the error to a
        # `ReviewError`; otherwise we let the original exception
        # propagate.
        try:
            apply_review(review_request["id"])
        except subprocess.CalledProcessError as err:
            error = err.output.decode("utf-8")
            if "patch does not apply" in error:
                raise ReviewError(error)

            raise err


def post_review(review_request, message):
    """Post a review on the review board."""
    print("Posting review on %s :\n%s" \
          % (review_request["absolute_url"], message))

    review_url = review_request["links"]["reviews"]["href"]
    data = urllib.parse.urlencode({"body_top": message, "public": "true"})
    api(review_url, data)


@atexit.register
def cleanup():
    """Clean the git repository."""
    try:
        shell("git clean -fd", SCRIPT_PATH)

        print(HEAD)
        shell("git reset --hard %s" % HEAD, SCRIPT_PATH)
    except subprocess.CalledProcessError as err:
        print("Failed command: %s\n\nError: %s" % (err.cmd, err.output))


def verify_review(review_request):
    """Verify a review."""
    print("Verifying review %s" % review_request["id"])
    build_output = "build_" + str(review_request["id"])

    try:
        # Recursively apply the review and its dependents.
        reviews = []
        apply_reviews(review_request, reviews)

        reviews.reverse()  # Reviews are applied in the reverse order.

        command = ""
        if platform.system() == 'Windows':
            command = "support\\windows-build.bat"

            # There is no equivalent to `tee` on Windows.
            subprocess.check_call(
                ['cmd', '/c',
                 '%s 2>&1 > %s' % (command, build_output)])
        else:
            # Launch docker build script.

            # TODO(jojy): Launch 'docker_build.sh' in subprocess so that
            # verifications can be run in parallel for various configurations.
            configuration = ("export "
                             "OS='ubuntu:14.04' "
                             "BUILDTOOL='autotools' "
                             "COMPILER='gcc' "
                             "CONFIGURATION='--verbose "
                             "--disable-libtool-wrappers "
                             "--disable-parallel-test-execution' "
                             "ENVIRONMENT='GLOG_v=1 MESOS_VERBOSE=1'")

            command = "%s; ./support/docker-build.sh" % configuration

            # `tee` the output so that the console can log the whole build
            # output. `pipefail` ensures that the exit status of the build
            # command ispreserved even after tee'ing.
            subprocess.check_call([
                'bash', '-c',
                ('set -o pipefail; %s 2>&1 | tee %s') % (command, build_output)
            ])

        # Success!
        post_review(
            review_request, "Patch looks great!\n\n"
            "Reviews applied: %s\n\n"
            "Passed command: %s" % (reviews, command))
    except subprocess.CalledProcessError as err:
        # If we are here because the docker build command failed, read the
        # output from `build_output` file. For all other command failures read
        # the output from `e.output`.
        if os.path.exists(build_output):
            output = open(build_output).read()
        else:
            output = err.output.decode(sys.stdout.encoding)

        if platform.system() == 'Windows':
            # We didn't output anything during the build (because `tee`
            # doesn't exist), so we print the output to stdout upon error.
            sys.stdout.buffer.write(output.encode())

        # Truncate the output when posting the review as it can be very large.
        if len(output) > REVIEW_SIZE:
            output = "...<truncated>...\n" + output[-REVIEW_SIZE:]

        if os.environ.get("BUILD_URL"):
            output += "\nFull log: "
            output += urllib.parse.urljoin(os.environ["BUILD_URL"], "console")

        post_review(
            review_request, "Bad patch!\n\n"
            "Reviews applied: %s\n\n"
            "Failed command: %s\n\n"
            "Error:\n%s" % (reviews, err.cmd, output))
    except ReviewError as err:
        post_review(
            review_request, "Bad review!\n\n"
            "Reviews applied: %s\n\n"
            "Error:\n%s" % (reviews, err.args[0]))

    # Clean up.
    cleanup()


def review_updated(review_request, review_time):
    """Returns whether this review request chain was updated after review
       time."""
    # If the latest diff on this review request was uploaded after the last
    # review from `USER`, we need to verify it.
    diffs_url = review_request["links"]["diffs"]["href"]
    diffs = api(diffs_url)
    diff_time = None
    if "diffs" in diffs:
        # Get the timestamp of the latest diff.
        timestamp = diffs["diffs"][-1]["timestamp"]
        diff_time = parse_time(timestamp)
        print("Latest diff timestamp: %s" % diff_time)

    if diff_time and review_time < diff_time:
        return True

    # If the latest dependency change on this review request happened after the
    # last review from `USER`, we need to verify it.
    changes_url = review_request["links"]["changes"]["href"]
    changes = api(changes_url)
    dependency_time = None
    for change in changes["changes"]:
        if "depends_on" in change["fields_changed"]:
            timestamp = change["timestamp"]
            dependency_time = parse_time(timestamp)
            print("Latest dependency change timestamp: %s" % dependency_time)
            break

    if dependency_time and review_time < dependency_time:
        return True

    # Recursively check if any of the dependent review requests need
    # verification.
    for review in review_request["depends_on"]:
        review_url = review["href"]
        print("Dependent review: %s " % review_url)
        if review_updated(api(review_url)["review_request"], review_time):
            return True

    return False


def needs_verification(review_request):
    """Returns whether this review request chain needs to be verified."""
    print("Checking if review: %s needs verification" % review_request["id"])

    # Skip if the review blocks another review.
    if review_request["blocks"]:
        print("Skipping blocking review %s" % review_request["id"])
        return False

    # Get the timestamp of the latest review from `USER`.
    reviews_url = review_request["links"]["reviews"]["href"]
    reviews = api(reviews_url + "?max-results=200")
    review_time = None
    for review in reversed(reviews["reviews"]):
        if review["links"]["user"]["title"] == USER:
            timestamp = review["timestamp"]
            review_time = parse_time(timestamp)
            print("Latest review timestamp: %s" % review_time)
            break

    # Verify the review request if no reviews were found from `USER`.
    if review_time is None:
        return True

    # Recursively check if any review requests in the chain were updated.
    return review_updated(review_request, review_time)


def write_review_ids(review_ids):
    """Write the IDs of the review requests that need verification."""
    print("%s review requests need verification" % len(review_ids))
    if OUT_FILE is not None:
        with open(OUT_FILE, 'w') as f:
            f.write('\n'.join(review_ids))
    else:
        print('\n'.join(review_ids))


def main():
    """Main function to verify the submitted reviews."""
    review_requests_url = \
        "%s/api/review-requests/%s" % (REVIEWBOARD_URL, QUERY_PARAMS)

    review_requests = api(review_requests_url)
    review_ids = []
    for review_request in reversed(review_requests["review_requests"]):
        if (NUM_REVIEWS == -1 or len(review_ids) < NUM_REVIEWS) and \
           needs_verification(review_request):
            if not SKIP_VERIFY:
                verify_review(review_request)
            review_ids.append(str(review_request["id"]))

    write_review_ids(review_ids)


if __name__ == '__main__':
    main()