blob: b2ed74ade9faf9d0c25b76851b763a4d88cde742 [file] [log] [blame]
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import sys
import subprocess
import glob
from random import choice
import string
from optparse import OptionParser
import subprocess
import shutil
import shlex
import socket
# squelch mysqldb spurious warnings
import warnings
warnings.simplefilter('ignore')
# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
# ---- We do this so cloud_utils can be looked up in the following order:
# ---- 1) Sources directory
# ---- 2) waf configured PYTHONDIR
# ---- 3) System Python path
for pythonpath in (
"@PYTHONDIR@",
os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
):
if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
# ---- End snippet of code ----
from cloud_utils import check_selinux, CheckFailed, resolves_to_ipv6
def runCmd(cmds):
process = subprocess.Popen(' '.join(cmds), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode != 0:
raise Exception(stderr)
return stdout.decode('utf-8')
class DBDeployer(object):
parser = None
options = None
args = None
serversetup = None
ip = None
user,password,host,port,rootuser,rootpassword = [None,None,None,None,None,None]
isDebug = False
mgmtsecretkey = None
dbsecretkey = None
encryptiontype = None
dbConfPath = r"@MSCONF@"
dbFilesPath = r"@SETUPDATADIR@"
dbDotProperties = {}
dbDotPropertiesIndex = 0
encryptionKeyFile = '@MSCONF@/key'
encryptionJarPath = '@COMMONLIBDIR@/lib/jasypt-1.9.3.jar'
success = False
magicString = 'This_is_a_magic_string_i_think_no_one_will_duplicate'
tmpMysqlFile = os.path.join(os.path.expanduser('~/'), 'cloudstackmysql.tmp.sql')
mysqlBinPath = None
def preRun(self):
def backUpDbDotProperties():
dbpPath = os.path.join(self.dbConfPath, 'db.properties')
copyPath = os.path.join(self.dbConfPath, 'db.properties.origin')
if os.path.isfile(dbpPath):
shutil.copy2(dbpPath, copyPath)
backUpDbDotProperties()
def postRun(self):
def cleanOrRecoverDbDotProperties():
dbpPath = os.path.join(self.dbConfPath, 'db.properties')
copyPath = os.path.join(self.dbConfPath, 'db.properties.origin')
if os.path.isfile(copyPath):
if not self.success:
shutil.copy2(copyPath, dbpPath)
os.remove(copyPath)
cleanOrRecoverDbDotProperties()
if os.path.exists(self.tmpMysqlFile):
os.remove(self.tmpMysqlFile)
def info(self, msg, result=None):
output = ""
if msg is not None:
output = "%-80s"%msg
if result is True:
output += "[ \033[92m%-2s\033[0m ]\n"%"OK"
elif result is False:
output += "[ \033[91m%-6s\033[0m ]\n"%"FAILED"
sys.stdout.write(output)
sys.stdout.flush()
def debug(self, msg):
msg = "DEBUG:%s"%msg
sys.stdout.write(msg)
sys.stdout.flush()
def putDbProperty(self, key, value):
if key in self.dbDotProperties:
(oldValue, index) = self.dbDotProperties[key]
self.dbDotProperties[key] = (value, index)
else:
self.dbDotProperties[key] = (value, self.dbDotPropertiesIndex)
self.dbDotPropertiesIndex += 1
def getDbProperty(self, key):
if key not in self.dbDotProperties:
return None
(value, index) = self.dbDotProperties[key]
return value
def runMysql(self, text, table, isRoot=False):
kwargs = {}
if not isRoot:
kwargs['user'] = self.user
if self.password != '': kwargs['passwd'] = self.password
else:
kwargs['user'] = self.rootuser
if self.rootpassword != '': kwargs['passwd'] = self.rootpassword
kwargs['port'] = self.port
kwargs['host'] = self.host
if self.mysqlBinPath is not None:
mysqlPath = '"' + self.mysqlBinPath + os.sep + "mysql" + '"'
else:
mysqlPath = "mysql"
try:
mysqlCmds = [mysqlPath, '--user=%s'%kwargs['user'], '--host=%s'%kwargs['host'], '--port=%s'%kwargs['port']]
if 'passwd' in kwargs:
mysqlCmds.append('--password=%s'%kwargs['passwd'])
open(self.tmpMysqlFile, 'w').write(text)
mysqlCmds.append('<')
mysqlCmds.append(self.tmpMysqlFile)
runCmd(mysqlCmds)
except Exception as e:
err = '''Encountering an error when executing mysql script
----------------------------------------------------------------------
table:
%s
Error:
%s
Sql parameters:
%s
----------------------------------------------------------------------
'''%(table, e.__str__(), kwargs)
self.errorAndExit(err)
def errorAndContinue(self, msg):
sys.stderr.write(msg)
sys.stderr.flush()
def errorAndExit(self, msg):
self.postRun()
err = '''\n\nWe apologize for below error:
***************************************************************
%s
***************************************************************
Please run:
cloudstack-setup-databases -h
for full help
''' % msg
sys.stderr.write(err)
sys.stderr.flush()
sys.exit(1)
def setupDBSchema(self):
if not self.rootuser:
self.info("No mysql root user specified, will not create Cloud DB schema\n", None)
return
replacements = (
("CREATE USER cloud identified by 'cloud';",
"CREATE USER %s@`localhost` identified by '%s'; CREATE USER %s@`%%` identified by '%s';"%(
(self.user,self.password,self.user,self.password)
)),
("cloud identified by 'cloud';",
"%s identified by '%s';"%(self.user,self.password)),
("cloud@`localhost` identified by 'cloud'",
"%s@`localhost` identified by '%s'"%(self.user,self.password)),
("cloud@`%` identified by 'cloud'",
"%s@`%%` identified by '%s'"%(self.user,self.password)),
("to cloud@`localhost`",
"to %s@`localhost`"%self.user),
("to cloud@`%`",
"to %s@`%%`"%self.user),
("TO cloud@`localhost`",
"to %s@`localhost`"%self.user),
("TO cloud@`%`",
"to %s@`%%`"%self.user),
("WHERE `User` = 'cloud' and host =",
"WHERE `User` = '%s' and host ="%self.user),
("DROP USER 'cloud'",
"DROP USER '%s'"%self.user),
("CALL `test`.`drop_user_if_exists`() ;",
""),
)
for f in ["create-database","create-schema", "create-database-premium","create-schema-premium"]:
p = os.path.join(self.dbFilesPath,"%s.sql"%f)
if not os.path.exists(p): continue
text = open(p).read()
for t, r in replacements: text = text.replace(t,r)
self.info("Applying %s"%p)
self.runMysql(text, p, True)
self.info(None, True)
if self.serversetup:
conf = os.path.join(self.dbConfPath, 'db.properties')
pcp = os.path.pathsep.join( glob.glob( os.path.join ( r"@PREMIUMJAVADIR@" , "*" ) ) )
systemjars = r"@SYSTEMJARS@".split()
try:
output = runCmd(['build-classpath'] + systemjars)
systemcp = output.strip()
except Exception as e:
systemcp = r"@SYSTEMCLASSPATH@"
mscp = r"@MSCLASSPATH@"
depscp = r"@DEPSCLASSPATH@"
classpath = os.path.pathsep.join([pcp,systemcp,depscp,mscp,conf])
try:
runCmd(["java","-cp",classpath,"com.cloud.test.DatabaseConfig", self.serversetup])
except Exception as e:
self.errorAndExit("Apply %s failed"%self.serversetup)
else:
p = os.path.join(self.dbFilesPath, 'server-setup.sql')
text = open(p).read()
self.info("Applying %s"%p)
self.runMysql(text, p, True)
self.info(None, True)
for f in ["templates"]:
p = os.path.join(self.dbFilesPath,"%s.sql"%f)
text = open(p).read()
self.info("Applying %s"%p)
self.runMysql(text, p, True)
self.info(None, True)
p = os.path.join(self.dbFilesPath,"schema-level.sql")
if os.path.isfile(p):
text = open(p).read()
self.info("Applying %s"%p)
self.runMysql(text, p, True)
self.info(None, True)
def prepareDBFiles(self):
def prepareDBDotProperties():
dbpPath = os.path.join(self.dbConfPath, 'db.properties')
dbproperties = open(dbpPath).read().splitlines()
newdbp = []
emptyLine = 0
for line in dbproperties:
passed = False
line = line.strip()
if line.startswith("#"): key = line; value = ''; passed = True
if line == '' or line == '\n': key = self.magicString + str(emptyLine); value = ''; emptyLine += 1; passed = True
try:
if not passed:
(key, value) = line.split('=', 1)
if key == "cluster.node.IP": value = self.ip
if key == "db.cloud.username": value = self.user
if key == "db.cloud.password": value = self.password
if key == "db.cloud.host": value = self.host
if key == "db.cloud.port": value = self.port
if key == "db.usage.username": value = self.user
if key == "db.usage.password": value = self.password
if key == "db.usage.host": value = self.host
if key == "db.usage.port": value = self.port
except Exception as e:
err = '''Wrong format in %s (%s):
Besides comments beginning "#" and empty line, all key-value pairs must be in formula of
key=value
for example:
db.cloud.username = cloud
''' % (dbpPath, line)
self.errorAndExit(err)
self.putDbProperty(key, value)
self.info("Preparing %s"%dbpPath, True)
self.putDbProperty("region.id", self.options.regionid)
prepareDBDotProperties()
def finalize(self):
def finalizeDbProperties():
entries = []
for key in list(self.dbDotProperties.keys()):
(value, index) = self.dbDotProperties[key]
if key.startswith("#"):
entries.insert(index, key)
elif key.startswith(self.magicString):
entries.insert(index, '')
else:
entries.insert(index, "%s=%s"%(key, value))
open(os.path.join(self.dbConfPath, 'db.properties'), 'w').write('\n'.join(entries))
self.info("Finalizing setup ...", None)
finalizeDbProperties()
self.info(None, True)
self.success = True # At here, we have done successfully and nothing more after this flag is set
def grabSystemInfo(self):
def getIpAddr():
try:
ip = socket.gethostbyname(socket.gethostname())
return ip
except Exception as e:
return "127.0.0.1"
if not self.ip:
self.ip = getIpAddr()
self.info("Detected local IP address as %s, will use as cluster management server node IP" % self.ip, True)
def checkSystemSetup(self):
def checkCloudDbFiles():
self.info("Checking Cloud database files ...", None)
dbfpaths = [ os.path.join(self.dbConfPath,"db.properties") ]
for dbf in dbfpaths:
if not os.path.exists(dbf):
self.errorAndExit("Cannot find %s"%dbf)
coreSchemas = ['create-database.sql', 'create-schema.sql', 'templates.sql']
if not self.serversetup:
coreSchemas.append('server-setup.sql')
checkingList = [os.path.join(self.dbFilesPath, x) for x in coreSchemas]
checkingList.append(self.encryptionJarPath)
for f in checkingList:
if not os.path.isfile(f):
self.errorAndExit("Cloud DB required file %s was not found"%f)
self.info(None, True)
def checkDbserverHostname():
self.info("Checking mysql server hostname ...", None)
if resolves_to_ipv6(self, self.port):
err = "%s resolves to an IPv6 address. The CloudStack does not support IPv6 yet.\nPlease fix this issue in either /etc/hosts or your DNS configuration.\n"%self.host
self.errorAndExit(err)
self.info(None, True)
def checkHostName():
self.info("Checking local machine hostname ...", None)
try:
output= socket.getfqdn()
except Exception as e:
err = "The host name of this computer does not resolve to an IP address.\nPlease use your operating system's network setup tools to fix this ('hostname --fqdn' %s).\n"%e.__str__()
self.errorAndExit(err)
self.info(None, True)
def checkSELinux():
self.info("Checking SELinux setup ...", None)
try:
check_selinux()
except CheckFailed as e:
self.info("checkSelinux failed with error continuing...", None)
self.errorAndContinue(e.__str__())
except OSError as e:
if e.errno == 2: pass
else:
self.info("checkSelinux failed with error continuing...", None)
self.errorAndContinue(e.__str__())
self.info(None, True)
checkCloudDbFiles()
checkHostName()
checkSELinux()
def processEncryptionStuff(self):
def encrypt(input):
cmd = ['java','-Djava.security.egd=file:/dev/urandom','-classpath','"' + self.encryptionJarPath + '"','org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI', 'encrypt.sh', 'input=%s'%input, 'password=%s'%self.mgmtsecretkey,'verbose=false']
return str(runCmd(cmd)).strip('\r\n')
def saveMgmtServerSecretKey():
if self.encryptiontype == 'file':
open(self.encryptionKeyFile, 'w').write(self.mgmtsecretkey)
def formatEncryptResult(value):
return 'ENC(%s)'%value
def encryptDBSecretKey():
self.putDbProperty('db.cloud.encrypt.secret', formatEncryptResult(encrypt(shlex.quote(self.dbsecretkey))))
def encryptDBPassword():
dbPassword = self.getDbProperty('db.cloud.password')
if dbPassword == '': return # Don't encrypt empty password
if dbPassword == None: self.errorAndExit('Cannot find db.cloud.password in %s'%os.path.join(self.dbConfPath, 'db.properties'))
self.putDbProperty('db.cloud.password', formatEncryptResult(encrypt(dbPassword)))
usagePassword = self.getDbProperty('db.usage.password')
if usagePassword == '': return # Don't encrypt empty password
if usagePassword == None: self.errorAndExit('Cannot find db.usage.password in %s'%os.path.join(self.dbConfPath, 'db.properties'))
self.putDbProperty('db.usage.password', formatEncryptResult(encrypt(usagePassword)))
self.info("Processing encryption ...", None)
self.putDbProperty("db.cloud.encryption.type", self.encryptiontype)
saveMgmtServerSecretKey()
encryptDBSecretKey()
encryptDBPassword()
self.info(None, True)
def parseOptions(self):
def parseOtherOptions():
if self.options.rootcreds:
self.rootuser,self.rootpassword = parseUserAndPassword(self.options.rootcreds)
if self.rootuser == self.user:
self.errorAndExit("--deploy-as= user name cannot be the user name supplied for the connection credentials")
self.info("Mysql root user name:%s"%self.rootuser, True)
self.info("Mysql root user password:%s"%"******", True)
if self.options.serversetup:
if not self.options.rootcreds:
self.errorAndExit("--auto= requires valid --deploy-as= credentials")
if os.path.isfile(self.options.serversetup):
self.errorAndExit("%s is not a valid file"%self.options.serversetup)
self.serversetup = self.options.serversetup
self.info("User specified server-setup.sql file at %s"%self.serversetup, True)
if self.options.mshostip:
self.ip = self.options.mshostip
self.info("Using specified cluster management server node IP %s" % self.options.mshostip, True)
self.encryptiontype = self.options.encryptiontype
self.mgmtsecretkey = self.options.mgmtsecretkey
self.dbsecretkey = self.options.dbsecretkey
self.isDebug = self.options.debug
if self.options.dbConfPath:
self.dbConfPath = self.options.dbConfPath
if self.options.dbFilesPath:
self.dbFilesPath = self.options.dbFilesPath
if self.options.encryptionKeyFile:
self.encryptionKeyFile = self.options.encryptionKeyFile
if self.options.encryptionJarPath:
self.encryptionJarPath = self.options.encryptionJarPath
if self.options.mysqlbinpath:
self.mysqlBinPath = self.options.mysqlbinpath
def parseUserAndPassword(cred):
stuff = cred.split(':')
if len(stuff) != 1 and len(stuff) != 2:
self.errorAndExit("Invalid user name and password format, must be in format of user:password (%s)"%cred)
user = stuff[0]
if len(user) < 1:
self.errorAndExit("Invalid user name and password format, must be in format of user:password, user name can not be empty")
if len(stuff) == 1:
password = ''
else:
password = stuff[1]
forbidden = "' \\`"
for f in forbidden:
if f in user: self.errorAndExit("User name cannot have the %r characters"%f)
if f in password: self.errorAndExit("Password cannot have the %r characters"%f)
return user, password
def parseCasualCredit():
def parseHostInfo(info):
stuff = info.split(":")
if len(stuff) == 1:
host = stuff[0]
port = 3306
elif len(stuff) == 2:
host = stuff[0]
try: port = int(stuff[1])
except ValueError: self.errorAndExit("The database port must be an integer (%s)"%stuff[1])
if port < 1: self.errorAndExit("The database port must be a positive integer (%s)"%stuff[1])
else:
self.errorAndExit("Invalid host and port format, it must be in format of host:port (%s)"%info)
return host, port
if len(self.args) == 0:
self.errorAndExit("Please specify user:password@hostname")
if len(self.args) > 1:
self.errorAndExit("There are more than one parameters for user:password@hostname (%s)"%self.args)
arg = self.args[0]
try:
try:
splitIndex = arg.rindex('@')
except ValueError:
# If it failed to find @, use host=localhost
splitIndex = len(arg)
arg += "@localhost"
finally:
stuff = [arg[:splitIndex], arg[splitIndex+1:]]
self.user,self.password = parseUserAndPassword(stuff[0])
self.host,self.port = parseHostInfo(stuff[1])
self.info("Mysql user name:%s"%self.user, True)
self.info("Mysql user password:%s"%"******", True)
self.info("Mysql server ip:%s"%self.host, True)
self.info("Mysql server port:%s"%self.port, True)
def validateParameters():
if self.encryptiontype != 'file' and self.encryptiontype != 'web':
self.errorAndExit('Wrong encryption type %s, --encrypt-type can only be "file" or "web'%self.encryptiontype)
#---------------------- option parsing and command line checks ------------------------
usage = """%prog user:[password]@mysqlhost:[port] [--deploy-as=rootuser:[rootpassword]] [--auto=/path/to/server-setup.xml] [-e ENCRYPTIONTYPE] [-m MGMTSECRETKEY] [-k DBSECRETKEY] [--debug]
This command sets up the CloudStack Management Server and CloudStack Usage Server database configuration (connection credentials and host information) based on the first argument.
If the the --deploy-as option is present, this command will also connect to the database using the administrative credentials specified as the value for the --deploy-as argument, construct the database environment needed to run the CloudStack Management Server, and alter the password specified for the user in the first argument. In this case, the user name specified in --deploy-as= cannot be the same as the user name specified for the connection credentials that the CloudStack Management Server will be set up with.
If a server-setup.xml cloud setup information file is specified with the --auto option, this command will also construct a customized database environment according to the cloud setup information in the file.
The port and the password are optional and can be left out.. If host is omitted altogether, it will default to localhost.
Examples:
%prog cloud:secret
sets user cloud and password 'secret' up in
@MSCONF@/db.properties, using localhost as the
database server
%prog sheng:rules@192.168.1.1
sets these credentials up in @MSCONF@/db.properties
%prog alex:founder@1.2.3.4 --deploy-as=root:nonsense
sets alex up as the MySQL user, then connects as the root user
with password 'nonsense', and recreates the databases, creating
the user alex with password 'founder' as necessary
%prog alex:founder@1.2.3.4 --deploy-as=root:nonsense -e file -m password -k dbpassword
In addition actions performing in above example, using 'password' as management server encryption key
and 'dbpassword' as database encryption key, saving management server encryption key to a file as the
encryption type specified by -e is file.
%prog alena:tests@5.6.7.8 --deploy-as=root:nonsense --auto=/root/server-setup.xml
sets alena up as the MySQL user, then connects as the root user
with password 'nonsense' to server 5.6.7.8, then recreates the
databases and sets up the alena user, then performs an automated
database setup using the information in server-setup.xml
"""
self.parser = OptionParser(usage=usage)
self.parser.add_option("-v", "--debug", action="store_true", dest="debug", default=False,
help="If enabled, print the commands it will run as they run")
self.parser.add_option("-d", "--deploy-as", action="store", type="string", dest="rootcreds", default="",
help="Colon-separated user name and password of a MySQL user with administrative privileges")
self.parser.add_option("-a", "--auto", action="store", type="string", dest="serversetup", default="",
help="Path to an XML file describing an automated unattended cloud setup")
self.parser.add_option("-e", "--encrypt-type", action="store", type="string", dest="encryptiontype", default="file",
help="Encryption method used for db password encryption. Valid values are file, web. Default is file.")
self.parser.add_option("-m", "--managementserver-secretkey", action="store", type="string", dest="mgmtsecretkey", default="password",
help="Secret key used to encrypt confidential parameters in db.properties. A string, default is password")
self.parser.add_option("-k", "--database-secretkey", action="store", type="string", dest="dbsecretkey", default="password",
help="Secret key used to encrypt sensitive database values. A string, default is password")
self.parser.add_option("-i", "--mshost", action="store", type="string", dest="mshostip", default="",
help="Cluster management server host IP. A string, by default it will try to detect a local IP")
self.parser.add_option("-r", "--regionid", action="store", type="string", dest="regionid", default="1",
help="Region Id for the management server cluster")
self.parser.add_option("-c", "--db-conf-path", action="store", dest="dbConfPath", help="The path to find db.properties which hold db properties")
self.parser.add_option("-f", "--db-files-path", action="store", dest="dbFilesPath", help="The path to find sql files to create initial database(s)")
self.parser.add_option("-j", "--encryption-jar-path", action="store", dest="encryptionJarPath", help="The path to the jasypt library to be used to encrypt the values in db.properties")
self.parser.add_option("-n", "--encryption-key-file", action="store", dest="encryptionKeyFile", help="The name of the file in which encryption key to be generated")
self.parser.add_option("-b", "--mysql-bin-path", action="store", dest="mysqlbinpath", help="The mysql installed bin path")
(self.options, self.args) = self.parser.parse_args()
parseCasualCredit()
parseOtherOptions()
validateParameters()
def run(self):
try:
self.preRun()
self.parseOptions()
self.checkSystemSetup()
self.grabSystemInfo()
self.prepareDBFiles()
self.setupDBSchema()
self.processEncryptionStuff()
self.finalize()
finally:
self.postRun()
print('')
print("CloudStack has successfully initialized database, you can check your database configuration in %s"%os.path.join(self.dbConfPath, 'db.properties'))
print('')
if __name__ == "__main__":
o = DBDeployer()
o.run()