#!/usr/bin/python
# ------------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
# ------------------------------------------------------------------------------

# Create changelist for any two branches/tags

# Prequisites:
# The old version has to be installed in the "madlib_old_vers" schema
# The new version has to be installed in the "madlib" (default) schema
# Two branches/tags must exist locally (run 'git fetch' to ensure you have the latest version)
# The current branch does not matter

# Usage (must be executed in the src/madpack directory):
# python create_changelist.py <database name> <old version branch> <new version branch> <changelist filename>
# If you are using the master branch, please make sure to edit the branch/tag in the output file

# Example (should be equivalent to changelist_1.13_1.14.yaml):
# python create_changelist.py madlib rel/v1.13 rel/v1.14 chtest1.yaml

import sys
import os

database = sys.argv[1]
old_vers = sys.argv[2]
new_vers = sys.argv[3]
ch_filename = sys.argv[4]

if os.path.exists(ch_filename):
    print "{0} already exists".format(ch_filename)
    raise SystemExit

err1 = os.system("""psql {0} -l > /dev/null""".format(database))
if err1 != 0:
    print "Database {0} does not exist".format(database)
    raise SystemExit

err1 = os.system("""psql {0} -c "select madlib_old_vers.version()" > /dev/null
                 """.format(database))
if err1 != 0:
    print "MADlib is not installed in the madlib_old_vers schema. Please refer to the Prequisites."
    raise SystemExit

err1 = os.system("""psql {0} -c "select madlib.version()" > /dev/null
                 """.format(database))
if err1 != 0:
    print "MADlib is not installed in the madlib schema. Please refer to the Prequisites."
    raise SystemExit

print "Creating changelist {0}".format(ch_filename)
os.system("""
    rm -f /tmp/madlib_tmp_nm.txt \
    /tmp/madlib_tmp_udf.txt \
    /tmp/madlib_tmp_udt.txt \
    /tmp/madlib_tmp_udo.txt \
    /tmp/madlib_tmp_udoc.txt \
    /tmp/madlib_tmp_typedep.txt \
    /tmp/madlib_tmp_typedep_udo.txt \
    /tmp/madlib_tmp_typedep_udoc.txt
    """)
try:
    # Find the new modules using the git diff
    err1 = os.system("git diff {old_vers} {new_vers} --name-only --diff-filter=A > /tmp/madlib_tmp_nm.txt".format(**locals()))
    if err1 != 0:
        print "Git diff failed. Please ensure that branches/tags are fetched."
        raise SystemExit

    f = open("/tmp/madlib_tmp_cl.yaml", "w")
    f.write(
"""# ------------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
# ------------------------------------------------------------------------------
""")

    f.write(
    """
# Changelist for MADlib version {old_vers} to {new_vers}

# This file contains all changes that were introduced in a new version of
# MADlib. This changelist is used by the upgrade script to detect what objects
# should be upgraded (while retaining all other objects from the previous version)

# New modules (actually .sql_in files) added in upgrade version
# For these files the sql_in code is retained as is with the functions in the
# file installed on the upgrade version. All other files (that don't have
# updates), are cleaned up to remove object replacements
""".format(**locals()))

    # Find the new .sql_in files that are not in test folders
    f.write("new module:\n")
    with open('/tmp/madlib_tmp_nm.txt') as fp:
        for line in fp:
            if 'sql_in' in line and '/test/' not in line:
                 f.write('    ' + line.split('/')[5].split('.')[0]+':\n')

    # Find the changed types and keep a list for future use
    os.system("psql {0} -f diff_udt.sql > /tmp/madlib_tmp_udt.txt".format(database))

    f.write("\n# Changes in the types (UDT) including removal and modification\n")
    f.write("udt:\n")
    udt_list=[]
    with open('/tmp/madlib_tmp_udt.txt') as fp:
        for line in fp:

            # Both the type and its array form shows up as separate types
            # We don't need both of them so we remove the array
            if 'UDT' in line and '[]' not in line:
                ch_type = line.split('|')[1].strip()
                udt_list.append(ch_type)
                f.write('    ' + ch_type +":\n")

    # Find the list of UDFs and UDAs
    # There are two main sources for these lists.
    # 1. The functions that actually got changed
    # 2. The functions that depend on a changed type

    # We will keep two lists (for UDF and UDA) and fill them as we parse the
    # output of diff functions

    udf_list=[]
    uda_list=[]
    current_list = udf_list

    # Find the changed functions/aggregates via the diff_udf script and write them to a file
    os.system("psql {0} -f diff_udf.sql > /tmp/madlib_tmp_udf.txt".format(database))

    # The entries in the file are ordered by type.
    # Read them line by line and add to the udf list until the first aggregate
    with open('/tmp/madlib_tmp_udf.txt') as fp:
        for line in fp:
            if 'type' in line:
                # When we get the first aggregate, we switch the current list
                if 'agg' in line:
                    current_list = uda_list
            if 'UDF' in line:
                current_list.append(line.split('|')[1])

    current_list = udf_list

    # Find the function that return a changed type
    # Note that we already ran the diff_udf.sql file
    # This means the get_functions() function is already defined in the database
    if os.path.exists("/tmp/madlib_tmp_typedep.txt"):
        os.system("rm /tmp/madlib_tmp_typedep.txt")
    os.system("touch /tmp/madlib_tmp_typedep.txt")
    for t in udt_list:

        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null """.format(**locals()))

        # Find all of the old functions that return this particular type t and write to a table
        os.system("""psql {database} -c "SELECT get_functions('__tmp__madlib__', 'madlib_old_vers', 'madlib_old_vers.{t}')" > /dev/null """.format(**locals()))

        # Order them in descending order on type so that we read the functions first and aggregates last
        os.system("""psql {database} -x -c "SELECT type, name, retype, argtypes FROM __tmp__madlib__ ORDER BY type DESC" > /tmp/madlib_tmp_typedep.txt """.format(**locals()))

        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null """.format(**locals()))

        with open('/tmp/madlib_tmp_typedep.txt') as fp:
            for line in fp:
                if '|' in line:
                    sp = line.split('|')

                    # Type is only used for switching the current list from udf to uda
                    if sp[0].strip() == 'type':
                        if sp[1].strip() == 'agg':
                            current_list = uda_list

                    # Add the functions with formatting
                    elif sp[0].strip() == 'name':
                        current_list.append('    - ' + sp[1].strip() + ":\n")
                    elif sp[0].strip() == 'retype':
                        current_list.append('        rettype: schema_madlib.' + sp[1].strip() + "\n")
                    elif sp[0].strip() == 'argtypes':
                        current_list.append('        argument: ' + sp[1].strip() + "\n")

    # Write the UDF and UDA lists to the changelist
    f.write(
    """
# List of the UDF changes that affect the user externally. This includes change
# in function name, return type, argument order or types, or removal of
# the function. In each case, the original function is as good as removed and a
# new function is created. In such cases, we should abort the upgrade if there
# are user views dependent on this function, since the original function will
# not be present in the upgraded version.
""")

    f.write("udf:\n")
    for line in udf_list:
        f.write(line)

    f.write("""
# Changes to aggregates (UDA) including removal and modification
# Overloaded functions should be mentioned separately
""")

    f.write("uda:\n")
    for line in uda_list:
        f.write(line)

    #TODO: We need to decide how we want to check for these
    f.write(
    """
# List of the UDC, UDO and UDOC changes.
""")
    f.write("udc:\n")

    # Find the changed operators

    f.write("\n# Changes in the operators (UDO)\n")
    f.write("udo:\n")

    os.system("psql {0} -f diff_udo.sql > /tmp/madlib_tmp_udo.txt".format(database))
    with open('/tmp/madlib_tmp_udo.txt') as fp:
        for line in fp:
            if ' | ' in line:
                    sp = line.split(' | ')

                    if sp[0].strip() == 'name':
                        f.write("    - '" + sp[1].strip() + "':\n")
                    elif sp[0].strip() == 'rettype':
                        f.write('        rettype: ' + sp[1].strip() + "\n")
                    elif sp[0].strip() == 'oprright':
                        f.write('        rightarg: ' + sp[1].strip() + "\n")
                    elif sp[0].strip() == 'oprleft':
                        f.write('        leftarg: ' + sp[1].strip() + "\n")

    for t in udt_list:

        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null """.format(**locals()))

        os.system("""psql {database} -c "SELECT get_udos('__tmp__madlib__', 'madlib_old_vers', '{t}')" > /dev/null """.format(**locals()))

        os.system("""psql {database} -x -c "SELECT name, rettype, oprright, oprleft FROM __tmp__madlib__ ORDER BY name DESC" > /tmp/madlib_tmp_typedep_udo.txt """.format(**locals()))

        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null """.format(**locals()))

        with open('/tmp/madlib_tmp_typedep_udo.txt') as fp:
            for line in fp:
                if '|' in line:
                    sp = line.split('|')

                    if sp[0].strip() == 'name':
                        f.write("    - '" + sp[1].strip() + "':\n")
                    elif sp[0].strip() == 'rettype':
                        f.write('        rettype: ' + sp[1].strip() + "\n")
                    elif sp[0].strip() == 'oprright':
                        f.write('        rightarg: ' + sp[1].strip() + "\n")
                    elif sp[0].strip() == 'oprleft':
                        f.write('        leftarg: ' + sp[1].strip() + "\n")


    # Find the changed operator classes

    f.write("\n# Changes in the operator classes (UDOC)\n")
    os.system("psql {0} -f diff_udoc.sql > /tmp/madlib_tmp_udoc.txt".format(database))
    f.write("udoc:\n")
    with open('/tmp/madlib_tmp_udoc.txt') as fp:
        for line in fp:
            if '|' in line:
                    sp = line.split('|')

                    if sp[0].strip() == 'opfamily_name':
                        f.write("    - '" + sp[1].strip() + "':\n")
                    elif sp[0].strip() == 'index_method':
                        f.write('        index_method: ' + sp[1].strip() + "\n")

    for t in udt_list:

        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null """.format(**locals()))

        os.system("""psql {database} -c "SELECT get_udocs('__tmp__madlib__', 'madlib_old_vers', '{t}')" > /dev/null """.format(**locals()))

        os.system("""psql {database} -x -c "SELECT opfamily_name, index_method FROM __tmp__madlib__ ORDER BY name DESC" > /tmp/madlib_tmp_typedep_udoc.txt """.format(**locals()))

        os.system("""psql {database} -c "DROP TABLE IF EXISTS __tmp__madlib__ " > /dev/null """.format(**locals()))

        with open('/tmp/madlib_tmp_typedep_udoc.txt') as fp:
            for line in fp:
                if '|' in line:
                    sp = line.split('|')

                    if sp[0].strip() == 'opfamily_name':
                        f.write("    - " + sp[1].strip() + ":\n")
                    elif sp[0].strip() == 'index':
                        f.write('        index: ' + sp[1].strip() + "\n")

    f.close()

    # Copy the new changelist file to its proper location
    # This helps to keep the madlib folder clean in case the program stops
    # unexpectedly
    os.system("cp /tmp/madlib_tmp_cl.yaml {0}".format(ch_filename))

except:
    print "Something went wrong! The changelist might be wrong/corrupted."
    raise
finally:
    os.system("""
        rm -f /tmp/madlib_tmp_nm.txt \
        /tmp/madlib_tmp_udf.txt \
        /tmp/madlib_tmp_udt.txt \
        /tmp/madlib_tmp_udo.txt \
        /tmp/madlib_tmp_udoc.txt \
        /tmp/madlib_tmp_typedep.txt \
        /tmp/madlib_tmp_typedep_udo.txt \
        /tmp/madlib_tmp_typedep_udoc.txt \
        /tmp/madlib_tmp_cl.yaml
        """)