madpy/madpack/madpack

#!/usr/bin/python
#
# Main Madpack installation driver executable.

import yaml
import os
import sys
import argparse
from argparse import RawTextHelpFormatter
import glob
import shutil
import madpy
import subprocess
from madpy.madpack import configyml
import madpy.madpack.mkmethods
import madpy.madpack.migrations


def __get_config(conf_dir):
    return madpy.madpack.configyml.get_config(conf_dir, False)
def __get_mig(conf_dir, mig_dir):
    return madpy.madpack.migrations.MadPackMigration(mig_dir, conf_dir)
    
    
## the prep() method does two things:
# 1) if needed, create the "scripts" subdirectory to hold the 
#    migration scripts, and copy Config.yml in there
# 2) generate a single MADPack migration script from the forward/backward
#    scripts in each of the MADlib methods specified in Config.yml, named
#    with version number from Version.yml
# @param targetdir directory to place migration script files
# @param conf_dir directory where we can find Config.yml
def prep(targetdir, conf_dir):
    version = configyml.get_version(madpy.__path__[0])
    if not os.path.exists(targetdir):
        # initialize scripts directory.
        os.mkdir(targetdir)
    if not os.path.exists(targetdir+"/Config.yml"):
        # copy the Config.yml file in there for subsequent use
        shutil.copyfile(configyml.get_configfile(conf_dir), 
                        targetdir+"/Config.yml")
    mig = __get_mig(conf_dir,targetdir)
    conf = __get_config(conf_dir)

    # sanity check: is there an existing script matching current version?
    version = configyml.get_version(madpy.__path__[0])
    matches = glob.glob(targetdir+"/"+"".join(["[0-9]" for i in range (0,mig.mig_number_len)])+"_"+version+".py")
    if len(matches) > 0:
        print "migration files matching version " + version + " already exist: " + str(matches)
        sys.exit(2)
        
    forwards = []
    backwards = []

    for m in conf['methods']:
        # XXX  We assume here that madlib and madpy are installed side-by-side 
        # in the same directory!
        mdir = madpy.__path__[0]+'/../madlib/' + m['name'] + '/src/' + m['port'] + '/'
        try:
            install = yaml.load(open(mdir+'Install.yml'))
        except:
            print "method " + m['name'] + " misconfigured: missing Install.yml file in " + mdir
            sys.exit(2)
        try:
            install['fw'] and install['bw']
        except:
            print "method " + m['name'] + " misconfigured: Install.yml misformatted"
            sys.exit(2)
        if install['fw'] != None:
            forwards.append(mdir+install['fw'])
        if install['bw'] != None:
            backwards.append(mdir+install['bw'])
    
    fname = mig.generate(targetdir, version, forwards, backwards)

## internal function sanity checks and installs by num or version.  
# @param scriptdir directory with migration script files
# @param num MADpack migration number
# @param version MADlib version number
# @param conf_dir directory where we can find Config.yml
def __install(scriptdir, num, version, conf_dir):
    if not os.path.isdir(scriptdir):
        print "directory " + scriptdir + " does not exist"
        exit(2)
    m = __get_mig(conf_dir,scriptdir)
    try:
        m.setup()
    except madpy.madpack.migrations.MadPackMigrationError:
        print "no schema named 'madlib' in database"
        exit(2)

    m.migrate(mignumber=num, migversion=version)

## roll scripts fw/bw by MADpack migration number
# @param scriptdir where we can find migration scripts
# @param n migration number to roll to
# @param conf_dir directory where we can find Config.yml
def install_number(scriptdir, n, conf_dir):
    __install(scriptdir,n,None,conf_dir)

## roll scripts fw/bw by MADlib version number
# @param scriptdir where we can find migration scripts
# @param v MADlib version to roll to
# @param conf_dir directory where we can find Config.yml
def install_version(scriptdir, v, conf_dir):
    __install(scriptdir,None,v,conf_dir)

## invoke migration sanity check    
# @param scriptdir where we can find migration scripts
# @param conf_dir directory where we can find Config.yml
def __sanity(scriptdir, conf_dir):
    m = __get_mig(conf_dir,scriptdir)
    try:
        m.sanity()
    except madpy.madpack.migrations.MadPackMigrationError:
        print "sanity check failure"
        exit(2)

    
## uninstall is the same as install version <0
# @param scriptdir where we can find migration scripts
# @param conf_dir directory where we can find Config.yml
def uninstall(scriptdir, conf_dir):
    install_number(scriptdir, -1, conf_dir)
 
# parse args and invoke appropriate code.
# try running "-h" for usage info.       
def main(argv):
    parser = argparse.ArgumentParser(description='Configure madlib package manager', 
                                     argument_default=False,
                                     formatter_class=RawTextHelpFormatter,
                                     epilog="""Command quickref: 
  install:   'make install' methods, run sql scripts to load into DB
  uninstall: run sql scripts to uninstall from DB, 'make uninstall' methods
  update:    migrate forward to latest script (same as '-e install')
  build:     'make install' methods, but do run sql scripts to load into DB
  clean:     'make clean' methods
  version:   print current version in DB
  sanity:    check if scriptdir files match migrationhistory in db""")
    parser.add_argument('command',
                        choices=['install','uninstall','update','build',
                                 'clean','version', 'sanity'], 
                        help="choice of commands")
    parser.add_argument('-i', '--install_version', nargs=1, dest='install_version',
                        metavar='VERSION',
                        help="migrate to version VERSION")
    parser.add_argument('-p', '--prep-only', dest='preponly',
                        action="store_true",
                        help="prepare scripts but do not install")
    parser.add_argument('-n', '--nomake', dest='nomake', 
                        action="store_true",
                        help="do not run 'make' on methods")
    parser.add_argument('-e', '--existing', dest='installonly', 
                        action="store_true",
                        help="use existing scripts, do not generate new scripts")
    parser.add_argument('-c', '--configdir', nargs=1, dest='configdir',
                        default=None,
                        help="""directory holding Config.yml
(default SCRIPTDIR if already initialized, else
"""
                              + madpy.__path__[0]+")")
    parser.add_argument('-s', '--scriptdir', nargs=1, dest='scriptdir',
                        default=madpy.__path__[0] + "/config/scripts",
                        help="""directory holding scripts 
(default """
                             + madpy.__path__[0] + "/config/scripts"
                             +")")
    parser.add_argument('-v', '--verbose', dest='verbose', 
                        action="store_true",
                        help="print more details during installation")

    args = parser.parse_args()
    
    # if config file left unspecified, first check scriptdir.
    # if nothing there, take the installed version: madpy/Config.yml
    if not args.configdir:
       if os.path.exists(args.scriptdir+"/Config.yml"):
           args.configdir = [args.scriptdir]
       else:
           args.configdir = madpy.__path__
    
    conf = configyml.get_config(args.configdir[0], False)
    
    # command 'update' a synonym for "-e install"
    if args.command == 'update':
        args.command = 'install'
        args.installonly = True
    
    if args.install_version and args.command != 'install':
        print "-i or --install_version flag only compatible with 'install' command"
        exit(2)
    if args.install_version:
        instv = args.install_version[0]
    else:
        instv = None
     
    if args.command == 'version':
        v = __get_mig(args.configdir[0],mig_dir = args.scriptdir).current_version() 
        if v == -1:
            print "no version found in database"
        else:
            print v
    elif args.command == 'uninstall':
        print "Uninstalling MADlib : " # + conf['connect_args'][1] + ", schema=" + conf['target_schema']
        if not args.nomake:
            madpy.madpack.mkmethods.make_methods('uninstall', conf, args.verbose)
        if not os.path.exists(args.scriptdir):
            print "scripts directory missing"
            sys.exit(2)
        uninstall(args.scriptdir, args.configdir[0])
        print "MADlib uninstalled OK."
    elif args.command == 'install':
        print "\n*** Installing MADlib: " + conf['connect_args'][1] + ", schema=" + conf['target_schema'] + ' ***\n'
        # run make clean_data_built and make install on methods 
        # unless preponly or nomake
        if not args.preponly and not args.nomake:
            madpy.madpack.mkmethods.make_methods('clean_data_built', conf, args.verbose)
            madpy.madpack.mkmethods.make_methods('install', conf, args.verbose)
        # prep script unless version given or installonly
        if not args.installonly and not args.install_version:        
            prep(args.scriptdir, args.configdir[0])
        # run install script unless preponly or makeonly
        if not args.preponly:
            # run a specific install extension
            if conf['post_hook'] != 'undefined':
                post_hook = args.configdir[0] + '/ext/' + conf['post_hook']
                if not os.path.isfile( post_hook):
                    print "post install hook script missing (" + post_hook + ")"
                    sys.exit(2)                
                print "Running post installation script: " + conf['post_hook']
                try:
                    subprocess.call(['python ' + post_hook], shell=True)
                except madpy.madpack.migrations.MadPackMigrationError:
                    print "error while running post installation script: " + conf['post_hook']
                    sys.exit(2)   
            # run db migration
            install_version(args.scriptdir, instv, args.configdir[0])
        print "\n*** MADlib installation finished OK ***\n"
    elif args.command == 'build':
        if not args.nomake:
            madpy.madpack.mkmethods.make_methods('install', conf, args.verbose)
    elif args.command == 'clean' and not args.nomake:
        madpy.madpack.mkmethods.make_methods('clean', conf, args.verbose)
    elif args.command == 'sanity':
        __sanity(args.scriptdir, args.configdir[0])
        
if __name__ == "__main__":
    main(sys.argv[1:])