blob: cbf5754d38bd72367dac57c8ee23bc9a42383740 [file] [log] [blame]
#!/usr/bin/python
# -----------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# -----------------------------------------------------------------------
#!/usr/bin/env python
import os
# common routines for ducc_post_install and db_create
def addToCp(cp, lib):
return cp + ':' + lib
def execute(CMD):
print CMD
return os.system(CMD)
# --------------------------------------------------------------------------------
# these next methods are used to parse a table returned from cqlsh into a
# - header
# - dictionary of values for each row
# parse the header into a list of names
def parse_header(header):
ret = []
parts = header.split('|')
for p in parts:
ret.append(p.strip())
return ret
# parse a single line into a dictionary with key from the header and value from the line
def parse_line(header, line):
parts = line.split('|')
ret = {}
for k, v in zip(header, parts):
ret[k] = v.strip()
return ret
# parse a set of lines returned from cqlsh into a header and a list of dictionaries, one per line
# header_id is a sting we use to positively identify a header line
def parse(lines, header_id):
ret = []
header = []
for l in lines:
l = l.strip()
# print '[]', l
if ( l == '' ):
continue
if ( '---' in l ):
continue;
if ( 'rows)' in l ):
continue;
if ( header_id in l ):
header = parse_header(l)
continue
ret.append(parse_line(header, l))
return header, ret
# given a header and a collection of lines parsed by the utilities above, print a
# mostly un-ugly listing of the table retults
def format(header, lines):
# calculate max column widths
hlens = {}
for k in header:
hlens[k] = len(k)
for line in lines:
if ( not hlens.has_key(k) ):
hlens[k] = len(line[k])
else:
hlens[k] = max(len(line[k]), hlens[k])
# create a format string from the widths
fmt = ''
for k in header:
fmt = fmt + ' %' + str(hlens[k]) + 's'
# first the header
print fmt % tuple(header)
# now the rows
for line in lines:
l = []
for k in header:
l.append(line[k])
print fmt % tuple(l)
return
# end of row parsing utilities
# --------------------------------------------------------------------------------
def stop_database(pidfile):
print "Stopping the database."
CMD = ['kill', '-TERM', '`cat ' + pidfile + '`']
CMD = ' '.join(CMD)
execute(CMD)
def manual_config(DUCC_HOME, DUCC_HEAD):
print ''
print 'To manually configure the database edit', DUCC_HOME + '/cassandra-server/conf/casssandra.yaml'
print 'to Insure every occurance of DUCC_HEAD is replaced with', DUCC_HEAD, 'and every occurance'
print 'of DUCC_HOME is replaced with', DUCC_HOME + '.'
print ''
print 'Note that one occurance of DUCC_HEAD will be quoted: you must preserve these quotes, e.g. as "' + DUCC_HEAD + '".'
def update_cassandra_config(DUCC_HOME, DUCC_HEAD):
# Read cassandra.yaml and change the things necessary to configure it correctly
config = DUCC_HOME + '/cassandra-server/conf/cassandra.yaml'
f = open(config)
lines = []
for line in f:
if ( line.startswith('listen_address:') ):
line = line.strip();
print 'Database host is configured at', line
if ( not DUCC_HEAD in line ):
print 'Must reconfigure listen_address to', DUCC_HEAD
parts = line.strip().split(':')
old = parts[1].strip()
ch_head = "sed -i.bak s'/" + old + "/" + DUCC_HEAD + "'/ " + config
os.system(ch_head)
def configure_database(DUCC_HOME, DUCC_HEAD, java, db_autostart=True, db_host=None, db_user=None, db_pw=None, db_replication=None):
# for cassandra:
# in ducc_runtime/cassandra-server/conf we need to update cassandra.yaml to establish
# the data directories and db connection addresses
# Note this is a bootstrap routine and doesn't try to use common code that may depend on
# things being initialized correctly.
if ( db_pw == None ):
db_pw = raw_input("Enter database password OR 'bypass' to bypass database support:")
if ( db_pw == '' ):
print "Must enter a DB password or 'bypass' to continue."
return False
if ( db_pw == 'bypass' ):
print 'Database support will be bypassed'
return True
if(db_host == None):
db_host = DUCC_HEAD
db_host = db_host.split()[0]
print "database host: "+str(db_host)
if( db_autostart ):
if ( os.path.exists(DUCC_HOME + "/state/database/data") ):
print 'Database is already defined in', DUCC_HOME + '/database', '- but will try to rebuild.'
update_cassandra_config(DUCC_HOME, DUCC_HEAD)
here = os.getcwd()
os.chdir(DUCC_HOME + "/cassandra-server")
pidfile = DUCC_HOME + '/state/cassandra.pid'
consfile = DUCC_HOME + '/state/cassandra.configure.console'
print 'Starting the database. This might take a few moments if it is the first time.'
CMD = "bin/cassandra -p "+ pidfile + " > "+consfile+" 2>&1";
os.system(CMD);
print "Database is started. Waiting for initialization";
os.chdir(here)
else:
print "Database is not auto-managed.";
# Now start the db and create the schema
CLASSPATH = ''
CLASSPATH = addToCp(CLASSPATH, DUCC_HOME + '/lib/cassandra/*')
CLASSPATH = addToCp(CLASSPATH, DUCC_HOME + '/lib/guava/*')
CLASSPATH = addToCp(CLASSPATH, DUCC_HOME + '/lib/apache-log4j/*')
CLASSPATH = addToCp(CLASSPATH, DUCC_HOME + '/lib/uima-ducc/*')
CLASSPATH = addToCp(CLASSPATH, DUCC_HOME + '/apache-uima/apache-activemq/lib/*')
os.environ['CLASSPATH'] = CLASSPATH
print os.environ['CLASSPATH']
ret = True
CMD = [java, '-DDUCC_HOME=' + DUCC_HOME, 'org.apache.uima.ducc.database.DbCreate', db_host, db_user, db_pw]
if(db_replication != None):
CMD.append(db_replication)
CMD = ' '.join(CMD)
if ( execute(CMD) == 0 ):
print 'Database is initialized.'
else:
print 'Database schema could not be defined.'
ret = False
if( db_autostart ):
stop_database(pidfile)
return ret
def update_database(DUCC_HOME, jvm):
CMD = [jvm, '-DDUCC_HOME=' + DUCC_HOME, 'org.apache.uima.ducc.database.DbUpdate']
CMD = ' '.join(CMD)
if ( execute(CMD) != 0 ):
print 'Database schema update failure.'
#else:
# print 'Database schema update success.'