blob: 6af6993e5e1a2d311ef50dab59f3d9e638a8160d [file] [log] [blame]
#!/usr/bin/env python
# -----------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# -----------------------------------------------------------------------
import os
import sys
import getopt
import platform
import string
import random
import shutil
import subprocess
from stat import *
# DuccUtil adds ../bin to the path so ducc_base can be found
from ducc_util import DuccUtil
from ducc_base import Properties
from ducc_base import find_ducc_home
from ducc_base import find_localhost
from ducc_base import which
from ducc import Ducc
import db_util as dbu
class PostInstall():
def usage(self, msg):
if ( msg != None ):
print ' '.join(msg)
print "Usage:"
print " ducc_post_install [options]"
print " If no options prompts are given for expected parameters."
print ""
print "Options:"
print " [-n, --head-node] <ducc head node>"
print " This is the name of the host that will run the DUCC management processes."
print ""
print " [-k, --keystore] <webserver keystore password>"
print " This is the password to be used to establish the webserver's keystore."
print ""
print " [-j, --jvm] <path to java executable>"
print " This is the full path to java command to be used to start DUCC; e.g., /usr/bin/java"
print ""
print " [-a, --db-automanage] <True|False>"
print " Specify False if DUCC should not start/stop database."
print ""
print " [-r, --db-replication] <integer>"
print " Specify database replication level (default = 1)."
print ""
print " [-m, --db-home] <path>"
print " The location of the database home directory, only specify when db-automanage is False and the default database directory should not be used."
print ""
print " [-o, --db-host-list] host1 <host2 host3...>"
print " The database host(s)."
print ""
print " [-u, --db-user] <root user for database>"
print " This is the user DUCC uses to manage the database."
print ""
print " [-d, --db-password] <root password for database>"
print " This is the password DUCC uses to manage the database."
print ""
print " [-b, --br-password] <password for broker>"
print " This is the password DUCC uses to manage the broker."
print ""
print " [-h, -? --help]"
print " Prints this message."
print ""
sys.exit(1)
def fail(self, *msg):
print ' '.join(msg)
print "POST INSTALLATION FAILED"
sys.exit(1)
def warn(self, *msg):
print ''
print 'WARNING'
print 'WARNING', ' '.join(msg)
print 'WARNING'
print ''
def addToCp(self, cp, lib):
return cp + ':' + self.DUCC_HOME + '/' + lib
def execute(self, CMD):
print CMD
rc = os.system(CMD)
if ( rc != 0 ):
print 'Failure, cannot continue.'
sys.exit(1)
# set username and password in broker credentials file
def configure_broker(self):
cf = self.DUCC_HOME+"/resources.private/ducc-broker-credentials.properties"
# create file if it does not exist
if(self.broker_pw == None):
self.broker_pw = self.generate_pw()
if ( not os.path.exists(cf) ):
# create file with username & password
print "broker configuration create"
with open(cf, 'w') as f:
line = 'ducc.broker.admin.username=admin'+'\n'
print line
f.write(line)
line = 'ducc.broker.admin.password='+self.broker_pw+'\n'
print line
f.write(line)
# update existing file
else:
# re-write file replacing username & password
print "broker configuration edit"
with open(cf, 'r+') as f:
lines = f.readlines()
f.seek(0)
f.truncate()
for line in lines:
if 'ducc.broker.admin.username=' in line:
line = 'ducc.broker.admin.username=admin'+'\n'
print line
if 'ducc.broker.admin.password=' in line:
line = 'ducc.broker.admin.password='+self.broker_pw+'\n'
print line
f.write(line)
return
def setup_database(self):
# for cassandra:
# in ducc_runtime/cassandra-server/conf we need to update cassandra.yaml to establish
# the data directories and db connection addresses
# Note this is a bootstrap routine and doesn't try to use common code that may depend on
# things being initialized correctly.
# If are re-running the DB may already have been created so use the saved password
db_pw = self.ducc_private_properties.get('db_password')
if (db_pw == None):
db_pw = self.get_pw(self.database_pw)
#if ( db_pw == 'bypass' ):
# print 'Database support will be disabled'
# self.update_property('ducc.database.host', '--disabled--', '# Database support is disabled')
# return True;
if ( dbu.configure_database(self.DUCC_HOME, self.ducc_head, self.path_to_java, self.database_automanage, self.database_host_list, self.database_user, db_pw, self.database_replication) ):
print 'Configuring DUCC to use the database.'
self.update_property('ducc.service.persistence.impl', 'org.apache.uima.ducc.database.StateServicesDb', '# Service manager persistence')
self.update_property('ducc.job.history.impl', 'org.apache.uima.ducc.database.HistoryManagerDb', '# History and checkpoint')
self.update_property('ducc.rm.persistence.impl', 'org.apache.uima.ducc.database.RmStatePersistence', '# RM state persistence')
self.ducc_private_properties.delete('db_password')
self.ducc_private_properties.put('db_password', db_pw, ['#Db password, default is randomly generated']);
self.ducc_private_properties.delete('db_password_guest')
self.ducc_private_properties.put('db_password_guest', db_pw, ['#Db password for guest, default is same value as db_password']);
return True
else:
return False
# generate a random string between 8 and 16 characters long
def generate_pw(self):
pwlen = random.randrange(8,16)
reply = ''.join([random.choice(string.ascii_letters + string.digits) for n in xrange(pwlen)])
return reply
# if password is not specified then generate a random one
def get_pw(self, given):
reply = given
if ( given == None ):
reply = self.generate_pw()
return reply
def create_keystore(self, keytool):
'''
CN - Common Name of the certificate owner
OU - Organizational Unit of the certificate owner
O - Organization to which the certificate owner belongs
L - Locality name of the certificate owner
S - State or province of the certificate owner
C - Country of the certificate owner
'''
keystore = self.DUCC_HOME + "/webserver/etc/keystore"
cmd = 'rm ' + keystore
os.system(cmd);
#/usr/bin/keytool
keystore_key = 'ducc.ws.port.ssl.pw'
self.default_keystore_prop = self.ducc_private_properties.get_property(keystore_key)
self.default_keystore_pw = self.default_keystore_prop.k
rc = 1
reply = ''
while ( rc != 0 ):
reply = self.get_pw(self.keystore_pw)
cmd = keytool
cmd += ' '
cmd += '-genkey'
cmd += ' '
cmd += '-noprompt'
cmd += ' '
cmd += '-alias jetty'
cmd += ' '
cmd += '-dname "CN=org.apache.uima.ducc, OU=uima.ducc, O=Apache, L=Wilmington, S=Delaware, C=USA"'
cmd += ' '
cmd += '-keyalg RSA'
cmd += ' '
cmd += '-validity 10000'
cmd += ' '
cmd += '-keystore ' + keystore
cmd += ' '
cmd += '-storepass '+ reply
cmd += ' '
cmd += '-keypass '+ reply
rc = os.system(cmd);
self.default_keystore_prop.v = reply
print 'keystore = ', keystore
#print 'keypass = ', reply
#print 'storepass = ', reply
# Setup and verify amq
# make sure verify_ducc is sufficient - maybe move some checks to there?
def update_property(self, key, val, comment):
self.ducc_site_properties.put(key, val, [comment])
def get_java_bindir(self):
if ( self.path_to_java == None ):
self.path_to_java = which('java')
if ( self.path_to_java == None ):
reply = ''
while ( reply == '' ):
reply = raw_input('Enter full path to the Java executable: ')
else:
prompt = '[' + self.path_to_java + ']'
reply = raw_input('Enter full path to the Java executable: ' + prompt)
if ( reply != '' ):
self.path_to_java = reply
# automagically fix // in path to java
self.path_to_java = self.path_to_java.replace('//', '/')
self.update_property('ducc.jvm', self.path_to_java, '# The full path to java')
print 'Initialized property "ducc.jvm" to', self.path_to_java
# We're going to do more checks here so we don't proceed with bogosities
proc = subprocess.Popen(self.path_to_java + ' -version', shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
lines = []
for line in proc.stdout:
lines.append(line.strip())
proc.wait()
rc = proc.returncode
for line in lines:
print "JAVA: " + line
vertoks = lines[0].split()
self.java_version = vertoks[-1]
if ( rc != 0 ):
self.fail("Requested java at '" + self.path_to_java +"' cannot be run.")
return os.path.dirname(self.path_to_java)
def set_java_home(self):
jvm = self.path_to_java
if ( platform.system() == 'Darwin' ):
self.jvm_home = "/Library/Java/Home"
else:
ndx = jvm.rindex('/')
ndx = jvm.rindex('/', 0, ndx)
self.jvm_home = jvm[:ndx]
os.environ['JAVA_HOME'] = self.jvm_home
def get_java_version(self):
return self.java_version
def get_java_keytool(self, bindir):
keytool = bindir + "/keytool"
if ( not os.path.exists(keytool) ):
self.fail("Cannot find keytool in ", bindir + '.', + "Is ducc.jvm configured correctly?")
return keytool
def check_nodes(self):
nodes = self.DUCC_HOME + "/resources/ducc.nodes"
self.mkbackup(nodes)
nf = open(nodes, 'w')
nf.write(self.localhost)
nf.close()
print "Initial", nodes, "created."
def merge_properties(self):
# first task, always, merge the properties so subsequent code can depend on their validity.
base_props = self.DUCC_HOME + '/resources/default.ducc.properties'
site_props = self.DUCC_HOME + '/resources/site.ducc.properties'
run_props = self.DUCC_HOME + '/resources/ducc.properties'
merger = self.DUCC_HOME + '/admin/ducc_props_manager'
CMD = [merger, '--merge', base_props, '--with', site_props, '--to', run_props]
CMD = ' '.join(CMD)
print 'Merging', base_props, 'with', site_props, 'into', run_props
os.system(CMD)
def setup_ducc_uid(self):
self.ducc_uid = os.environ['LOGNAME']
self.update_property('ducc.uid', self.ducc_uid, "# ducc.uid is the user that runs the main DUCC daemons");
print "Ducc uid is configured as", self.ducc_uid, '\n'
def setup_ducc_head(self):
if ( self.ducc_head == None ):
self.ducc_head = self.localhost
reply = raw_input('Enter hostname of DUCC head[' + self.ducc_head + ']')
if ( reply != '' ):
self.ducc_head = reply
self.update_property('ducc.head', self.ducc_head, "# ducc.head is the node where the main DUCC daemons run");
print "Ducc head is configured as", self.ducc_head, '\n'
def setup_ducc_database(self):
# ducc.database.automanage
if(self.database_automanage == None):
if ( self.database_host_list == None ):
# db automanage default to T when db host list not specified
self.database_automanage = True
else:
# db automanage default to F when db host list specified
self.database_automanage = False
if(self.database_automanage == False):
self.update_property('ducc.database.automanage', 'False', '# Database start/stop not managed by DUCC')
else:
self.update_property('ducc.database.automanage', 'True', '# Database start/stop managed by DUCC')
# ducc.database.host.list
if ( self.database_host_list == None ):
self.database_host_list = self.ducc_head
self.update_property('ducc.database.host.list', self.database_host_list, '# Database host list')
# ducc.database.user
if ( self.database_user == None ):
self.database_user = 'ducc'
self.update_property('ducc.database.user', self.database_user, '# Database user')
# ducc.database.home
if ( self.database_home != None ):
if(self.database_automanage == False):
self.update_property('ducc.database.home', self.database_home, '# Database home')
else:
print "db-home ignored", self.database_home, '\n'
def mkbackup(self, fn):
if ( os.path.exists(fn) ):
bak = fn + '.bak'
shutil.move(fn, bak)
print 'Existing', fn, 'moved to', bak
def verify_permissions(self):
# should have 755 permissions
spot_checked_directories = ['../bin', '../lib', '../resources' ]
# should have 755 permissions
spot_checked_execs = ['../bin/ducc_submit']
# should have 644 permissions
spot_checked_data = ['../lib/uima-ducc-cli.jar', '../resources/default.ducc.properties']
ret = True
for f in spot_checked_directories:
if ( not os.path.exists(f) ):
print 'ERROR: Directory', f, 'cannot be found.'
ret = False
continue
stat = os.stat(f)
mode = oct(stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
expected = oct(0755)
if ( mode != expected ):
print 'ERROR: Directory', f, 'has permissions', mode, 'expected', expected
ret = False
for f in spot_checked_execs:
if ( not os.path.exists(f) ):
print 'ERROR: File', f, 'cannot be found.'
ret = False
continue
stat = os.stat(f)
mode = oct(stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
expected = oct(0755)
if ( mode != expected ):
print 'ERROR: File', f, 'has permissions', mode, 'expected', expected
ret = False
for f in spot_checked_data:
if ( not os.path.exists(f) ):
print 'ERROR: File', f, 'cannot be found.'
ret = False
continue
stat = os.stat(f)
mode = oct(stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
expected = oct(0644)
if ( mode != expected ):
print 'ERROR: File', f, 'has permissions', mode, 'expected', expected
ret = False
return ret
def verify_prereqs(self):
fn = '/usr/bin/nslookup'
# will throw exception if not found
dir_stat = os.stat(fn)
def main(self, argv):
self.verify_prereqs()
self.DUCC_HOME = find_ducc_home()
self.localhost = find_localhost()
cwd = os.getcwd()
if cwd != self.DUCC_HOME+'/admin':
print '>>> ERROR - this script must be run from the admin directory'
sys.exit(99)
print 'Using DUCC HOME:', self.DUCC_HOME, '\n'
if ( not self.verify_permissions() ):
print '--------------------------------------------------------------------------------'
print 'Package verificaiton fails. Most likely cause is an unexpected UMASK unpacking the distribution.'
print 'To unpack the distribution your UMASK must be set to 022.'
print ''
print 'Example:'
print ''
print 'umask 022; tar -xf [distribution]'
print '--------------------------------------------------------------------------------'
sys.exit(1)
self.ducc_uid = None
self.ducc_head = None
self.keystore_pw = None
self.database_automanage = None
self.database_replication = None
self.database_home = None
self.database_host_list = None
self.database_user = None
self.database_pw = None
self.broker_pw = None
self.path_to_java = None
try:
opts, args = getopt.getopt(argv, 'a:r:m:o:u:d:b:j:k:n:h?', ['db-automanage=', 'db-replication=', 'db-home=', 'db-host-list=', 'db-user=', 'db-password=', 'br-password=', 'jvm=', 'keystore=', 'head-node=', 'help'])
except:
self.usage("Invalid arguments " + ' '.join(argv))
for ( o, a ) in opts:
if o in ('-n', '--head-node'):
self.ducc_head = a
if o in ('-a', '--db-automanage'):
if(o in [ 'True', 'T', 'true', 't']):
self.database_automanage = True
else:
self.database_automanage = False
if o in ('-r', '--db-replication'):
self.database_replication = a
if o in ('-m', '--db-home'):
self.database_home = a
if o in ('-o', '--db-host-list'):
self.database_host_list = a
if o in ('-u', '--db-user'):
self.database_user = a
if o in ('-d', '--db-password'):
self.database_pw = a
if o in ('-b', '--br-password'):
self.broker_pw = a
if o in ('-k', '--keystore'):
self.keystore_pw = a
if o in ('-j', '--jvm'):
self.path_to_java = a
elif o in ('-h', '-?', '--help'):
self.usage(None)
resources = self.DUCC_HOME + '/resources'
self.site_properties_name = resources + '/site.ducc.properties'
self.ducc_properties = resources + '/ducc.properties'
if(os.path.exists(self.ducc_properties)):
text0 = '\nDUCC is already installed!'
text1 = '\nTo re-run ducc_post install be sure DUCC is stopped using "stop_ducc --all"'
text2 = "\nand back-up any DUCC configuration files that will be re-used,"
text3 = '\nthen re-install DUCC.'
self.fail(text0+text1+text2+text3)
self.mkbackup(self.site_properties_name)
self.mkbackup(self.ducc_properties)
print 'Python version:'
print sys.version
keystore_properties_name = self.DUCC_HOME + '/resources.private/ducc.private.properties'
self.ducc_private_properties = Properties()
self.ducc_private_properties.load(keystore_properties_name)
self.ducc_site_properties = Properties()
py_version = sys.version_info
if ( (py_version[0] != 2) or (py_version[1] < 4) ):
self.fail("\nPython must be installed at level 2.4 or higher.")
self.setup_ducc_uid()
self.setup_ducc_head()
self.setup_ducc_database()
self.check_nodes()
# insure java is configured and installed
self.java_bindir = self.get_java_bindir()
self.set_java_home()
print "java_home", os.environ['JAVA_HOME']
# As of DUCC 2.0, always set here on installation
print 'ActiveMQ is automanaged on node ', self.localhost
statedir = self.DUCC_HOME + "/state"
logdir = self.DUCC_HOME + "/logs"
#logwsdir = self.DUCC_HOME + "/logs/webserver"
historydir = self.DUCC_HOME + "/history"
if ( not os.path.exists(statedir) ):
os.mkdir(statedir)
if ( not os.path.exists(logdir) ):
os.mkdir(logdir)
#if ( not os.path.exists(logwsdir) ):
# os.mkdir(logwsdir)
if ( not os.path.exists(historydir) ):
os.mkdir(historydir)
# configure the AMQ broker
self.configure_broker()
# configure the database for local system and initialize the schema
if not self.setup_database():
print 'Database creation failed - DUCC setup incomplete'
sys.exit(1)
self.keytool = self.get_java_keytool(self.java_bindir)
print 'Java version:', self.get_java_version()
print 'Java is verified.'
self.create_keystore(self.keytool)
print '\nWeb server keystore generated from ducc.properties'
ws_duccbook = self.DUCC_HOME + "/webserver/root/system.duccbook.html"
if ( not os.path.lexists(ws_duccbook) ):
duccbook = self.DUCC_HOME + "/docs/book.html"
os.symlink(duccbook, ws_duccbook)
print '\nDUCC book installed into webserver root\n'
# set up the local properties, required to build ducc_ling
self.ducc_private_properties.write(keystore_properties_name)
self.ducc_site_properties.write(self.site_properties_name)
self.merge_properties()
# Make duccling
rc = os.system(self.DUCC_HOME + '/admin/build_duccling')
if ( rc != 0 ):
print 'Could not build ducc_ling. Run the command'
print ' build_duccling'
print 'to complete the installation (it must run without error).'
sys.exit(1)
print 'Initial DUCC setup complete.'
if __name__ == "__main__":
os.environ['DUCC_POST_INSTALL'] = 'DUCC_POST_INSTALL'
postinstall = PostInstall()
postinstall.main(sys.argv[1:])