| #!/usr/bin/env python |
| |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| |
| import os,logging,sys |
| from optparse import OptionParser |
| import mysql.connector |
| import subprocess |
| import glob |
| |
| # ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ---- |
| # ---- We do this so cloud_utils can be looked up in the following order: |
| # ---- 1) Sources directory |
| # ---- 2) waf configured PYTHONDIR |
| # ---- 3) System Python path |
| for pythonpath in ( |
| "@PYTHONDIR@", |
| os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"), |
| ): |
| if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath) |
| # ---- End snippet of code ---- |
| from cloud_utils import check_selinux, CheckFailed, resolves_to_ipv6 |
| import cloud_utils |
| |
| # RUN ME LIKE THIS |
| # python setup/bindir/cloud-migrate-databases.in --config=client/conf/override/db.properties --resourcedir=setup/db --dry-run |
| # --dry-run makes it so the changes to the database in the context of the migrator are rolled back |
| |
| # This program / library breaks down as follows: |
| # high-level breakdown: |
| # the module calls main() |
| # main processes command-line options |
| # main() instantiates a migrator with a a list of possible migration steps |
| # migrator discovers and topologically sorts migration steps from the given list |
| # main() run()s the migrator |
| # for each one of the migration steps: |
| # the migrator instantiates the migration step with the context as first parameter |
| # the instantiated migration step saves the context onto itself as self.context |
| # the migrator run()s the instantiated migration step. within run(), self.context is the context |
| # the migrator commits the migration context to the database (or rollsback if --dry-run is specified) |
| # that is it |
| |
| # The specific library code is in cloud_utils.py |
| # What needs to be implemented is MigrationSteps |
| # Specifically in the FromInitialTo21 evolver. |
| # What Db20to21MigrationUtil.java does, needs to be done within run() of that class |
| # refer to the class docstring to find out how |
| # implement them below |
| |
| class CloudContext(cloud_utils.MigrationContext): |
| def __init__(self,host,port,username,password,database,configdir,resourcedir): |
| self.host = host |
| self.port = port |
| self.username = username |
| self.password = password |
| self.database = database |
| self.configdir = configdir |
| self.resourcedir = resourcedir |
| self.conn = mysql.connector.connect(host=self.host, |
| user=self.username, |
| password=self.password, |
| database=self.database, |
| port=self.port) |
| self.conn.autocommit(False) |
| self.db = self.conn.cursor() |
| def wrapex(func): |
| sqlogger = logging.getLogger("SQL") |
| def f(stmt,parms=None): |
| if parms: sqlogger.debug("%s | with parms %s",stmt,parms) |
| else: sqlogger.debug("%s",stmt) |
| return func(stmt,parms) |
| return f |
| self.db.execute = wrapex(self.db.execute) |
| |
| def __str__(self): |
| return "CloudStack %s database at %s"%(self.database,self.host) |
| |
| def get_schema_level(self): |
| return self.get_config_value('schema.level') or cloud_utils.INITIAL_LEVEL |
| |
| def set_schema_level(self,l): |
| self.db.execute( |
| "INSERT INTO configuration (category,instance,component,name,value,description) VALUES ('Hidden', 'DEFAULT', 'database', 'schema.level', %s, 'The schema level of this database') ON DUPLICATE KEY UPDATE value = %s", (l,l) |
| ) |
| self.commit() |
| |
| def commit(self): |
| self.conn.commit() |
| #self.conn.close() |
| |
| def close(self): |
| self.conn.close() |
| |
| def get_config_value(self,name): |
| self.db.execute("select value from configuration where name = %s",(name,)) |
| try: return self.db.fetchall()[0][0] |
| except IndexError: return |
| |
| def run_sql_resource(self,resource): |
| sqlfiletext = file(os.path.join(self.resourcedir,resource)).read(-1) |
| sqlstatements = sqlfiletext.split(";") |
| for stmt in sqlstatements: |
| if not stmt.strip(): continue # skip empty statements |
| self.db.execute(stmt) |
| |
| |
| class FromInitialTo21NewSchema(cloud_utils.MigrationStep): |
| def __str__(self): return "Altering the database schema" |
| from_level = cloud_utils.INITIAL_LEVEL |
| to_level = "2.1-01" |
| def run(self): self.context.run_sql_resource("schema-20to21.sql") |
| |
| class From21NewSchemaTo21NewSchemaPlusIndex(cloud_utils.MigrationStep): |
| def __str__(self): return "Altering indexes" |
| from_level = "2.1-01" |
| to_level = "2.1-02" |
| def run(self): self.context.run_sql_resource("index-20to21.sql") |
| |
| class From21NewSchemaPlusIndexTo21DataMigratedPart1(cloud_utils.MigrationStep): |
| def __str__(self): return "Performing data migration, stage 1" |
| from_level = "2.1-02" |
| to_level = "2.1-03" |
| def run(self): self.context.run_sql_resource("data-20to21.sql") |
| |
| class From21step1toTo21datamigrated(cloud_utils.MigrationStep): |
| def __str__(self): return "Performing data migration, stage 2" |
| from_level = "2.1-03" |
| to_level = "2.1-04" |
| |
| def run(self): |
| systemjars = "@SYSTEMJARS@".split() |
| pipe = subprocess.Popen(["build-classpath"]+systemjars,stdout=subprocess.PIPE) |
| systemcp,throwaway = pipe.communicate() |
| systemcp = systemcp.strip() |
| if pipe.wait(): # this means that build-classpath failed miserably |
| systemcp = "@SYSTEMCLASSPATH@" |
| pcp = os.path.pathsep.join( glob.glob( os.path.join ( "@PREMIUMJAVADIR@" , "*" ) ) ) |
| mscp = "@MSCLASSPATH@" |
| depscp = "@DEPSCLASSPATH@" |
| migrationxml = "@SERVERSYSCONFDIR@" |
| conf = self.context.configdir |
| cp = os.path.pathsep.join([pcp,systemcp,depscp,mscp,migrationxml,conf]) |
| cmd = ["java"] |
| cmd += ["-cp",cp] |
| cmd += ["com.cloud.migration.Db20to21MigrationUtil"] |
| logging.debug("Running command: %s"," ".join(cmd)) |
| subprocess.check_call(cmd) |
| |
| class From21datamigratedTo21postprocessed(cloud_utils.MigrationStep): |
| def __str__(self): return "Postprocessing migrated data" |
| from_level = "2.1-04" |
| to_level = "2.1" |
| def run(self): self.context.run_sql_resource("postprocess-20to21.sql") |
| |
| class From21To213(cloud_utils.MigrationStep): |
| def __str__(self): return "Dropping obsolete indexes" |
| from_level = "2.1" |
| to_level = "2.1.3" |
| def run(self): self.context.run_sql_resource("index-212to213.sql") |
| |
| class From213To22data(cloud_utils.MigrationStep): |
| def __str__(self): return "Migrating data" |
| from_level = "2.1.3" |
| to_level = "2.2-01" |
| def run(self): self.context.run_sql_resource("data-21to22.sql") |
| |
| class From22dataTo22(cloud_utils.MigrationStep): |
| def __str__(self): return "Migrating indexes" |
| from_level = "2.2-01" |
| to_level = "2.2" |
| def run(self): self.context.run_sql_resource("index-21to22.sql") |
| |
| # command line harness functions |
| |
| def setup_logging(level): |
| l = logging.getLogger() |
| l.setLevel(level) |
| h = logging.StreamHandler(sys.stderr) |
| l.addHandler(h) |
| |
| |
| def setup_optparse(): |
| usage = \ |
| """%prog [ options ... ] |
| |
| This command migrates the CloudStack database.""" |
| parser = OptionParser(usage=usage) |
| parser.add_option("-c", "--config", action="store", type="string",dest='configdir', |
| default=os.path.join("@MSCONF@"), |
| help="Configuration directory with a db.properties file, pointing to the CloudStack database") |
| parser.add_option("-r", "--resourcedir", action="store", type="string",dest='resourcedir', |
| default="@SETUPDATADIR@", |
| help="Resource directory with database SQL files used by the migration process") |
| parser.add_option("-d", "--debug", action="store_true", dest='debug', |
| default=False, |
| help="Increase log level from INFO to DEBUG") |
| parser.add_option("-e", "--dump-evolvers", action="store_true", dest='dumpevolvers', |
| default=False, |
| help="Dump evolvers in the order they would be executed, but do not run them") |
| #parser.add_option("-n", "--dry-run", action="store_true", dest='dryrun', |
| #default=False, |
| #help="Run the process as it would normally run, but do not commit the final transaction, so database changes are never saved") |
| parser.add_option("-f", "--start-at-level", action="store", type="string",dest='fromlevel', |
| default=None, |
| help="Rather than discovering the database schema level to start from, start migration from this level. The special value '-' (a dash without quotes) represents the earliest schema level") |
| parser.add_option("-t", "--end-at-level", action="store", type="string",dest='tolevel', |
| default=None, |
| help="Rather than evolving the database to the most up-to-date level, end migration at this level") |
| return parser |
| |
| |
| def main(*args): |
| """The entry point of this program""" |
| |
| parser = setup_optparse() |
| opts, args = parser.parse_args(*args) |
| if args: parser.error("This command accepts no parameters") |
| |
| if opts.debug: loglevel = logging.DEBUG |
| else: loglevel = logging.INFO |
| setup_logging(loglevel) |
| |
| # FIXME implement |
| opts.dryrun = False |
| |
| configdir = opts.configdir |
| resourcedir = opts.resourcedir |
| |
| try: |
| props = cloud_utils.read_properties(os.path.join(configdir,'db.properties')) |
| except (IOError,OSError),e: |
| logging.error("Cannot read from config file: %s",e) |
| logging.error("You may want to point to a specific config directory with the --config= option") |
| return 2 |
| |
| if not os.path.isdir(resourcedir): |
| logging.error("Cannot find directory with SQL files %s",resourcedir) |
| logging.error("You may want to point to a specific resource directory with the --resourcedir= option") |
| return 2 |
| |
| host = props["db.cloud.host"] |
| port = int(props["db.cloud.port"]) |
| username = props["db.cloud.username"] |
| password = props["db.cloud.password"] |
| database = props["db.cloud.name"] |
| |
| # tell the migrator to load its steps from the globals list |
| migrator = cloud_utils.Migrator(globals().values()) |
| |
| if opts.dumpevolvers: |
| print "Evolution steps:" |
| print " %s %s %s"%("From","To","Evolver in charge") |
| for f,t,e in migrator.get_evolver_chain(): |
| print " %s %s %s"%(f,t,e) |
| return |
| |
| #initialize a context with the read configuration |
| context = CloudContext(host=host,port=port,username=username,password=password,database=database,configdir=configdir,resourcedir=resourcedir) |
| try: |
| try: |
| migrator.run(context,dryrun=opts.dryrun,starting_level=opts.fromlevel,ending_level=opts.tolevel) |
| finally: |
| context.close() |
| except (cloud_utils.NoMigrationPath,cloud_utils.NoMigrator),e: |
| logging.error("%s",e) |
| return 4 |
| |
| if __name__ == "__main__": |
| retval = main() |
| if retval: sys.exit(retval) |
| else: sys.exit() |