| #! /usr/bin/env python |
| # ----------------------------------------------------------------------- |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # ----------------------------------------------------------------------- |
| |
| # ------------------------------------------------------------------------------------- |
| # Updates an existing ducc installation (1.1.0 or newer) from a binary tar.gz build |
| # Updates in place so preserves all the site-specific files |
| # Archives the current build in a timestamped direcory under DUCC_HOME/../ducc_archives |
| # - checks that ducc is not running |
| # - creates a site.ducc.properties file if updating from DUCC 1.1.0 |
| # - creates a time-stamped archive directory to hold the current build |
| # - archives files replaced by the new build, except |
| # - retains the customized files and any added to resources & resources.private |
| # - retains the webserver/etc/keystore file |
| # - retains the customizable files: admin/local_hooks.py webserver/root/site.jsp webserver/root/js/ducc.local.js |
| # - retains other files added to certain directories, i.e. lib admin |
| # ebserver/root webserver/root/$banner webserver/root/js webserver/root/opensources/images |
| # - rebuilds the non-privileged ducc_ling |
| # |
| # Note: files added to other directories will not be retained, but will be archived |
| # new versions of files not replaced are left in the extract for comparison |
| # |
| # To revert back to an archived build: |
| # - copy/move all of the archived files back to the runtime |
| # ------------------------------------------------------------------------------------- |
| import os |
| import sys |
| import datetime |
| import fnmatch |
| import re |
| import shutil |
| |
| global preserveFiles |
| global preserveDirectories |
| global modifiableDirectories |
| global lenRuntime |
| |
| def usage(): |
| print "Usage: ducc_update ducc_runtime ducc-binary-tarfile" |
| print "" |
| print " Installs or updates DUCC at 'ducc_runtime' from the build in 'ducc-binary-tarfile'" |
| print " If there is no existing runtime a fresh installation will be placed in 'ducc_runtime'" |
| print " If the runtime exists, the update will:" |
| print " - check that DUCC is not running" |
| print " - create a site.ducc.properties file if updating from DUCC 1.1.0" |
| print " - create a time-stamped archive directory to hold the old runtime" |
| print " - archive current files before updating them, except for the customizable ones" |
| print " - report which are replaced, added, or kept" |
| print " - leave in place any added to certain directories" |
| print " - rebuild the non-privileged ducc_ling" |
| print " Note: only files added to the following directories will be retained, others will be archived:" |
| print modifiableDirectories |
| print "" |
| print " To revert back to an archived build:" |
| print " - copy/move all of the archived files back to the runtime" |
| |
| #----------------------------------------------------------------------------------------- |
| # Get the version of the build from the name of the cli jar |
| #----------------------------------------------------------------------------------------- |
| def get_oversion(runtime): |
| fns = fnmatch.filter(os.listdir(runtime + '/lib/uima-ducc'), 'uima-ducc-cli-*.jar') |
| if ( len(fns) == 0 ): |
| raise Exception("Not a valid DUCC installation - missing versioned cli jar") |
| m = re.match('uima\\-ducc\\-cli\\-([a-zA-Z0-9_\\.\\-]+)\\.jar', fns[0]) |
| if ( m == None ): |
| raise Exception("Not a valid DUCC installation - invalid name: " + fns[0]) |
| return m.group(1) |
| |
| #----------------------------------------------------------------------------------------- |
| # Get the version of the build from the name of the tar file |
| #----------------------------------------------------------------------------------------- |
| def get_nversion(tarfile): |
| p = re.compile('^uima\\-ducc\\-([a-zA-Z0-9_\\.\\-]+)\\-bin\\.tar\\.gz$') |
| m = p.search(tarfile, 0) |
| if ( m == None ): |
| raise Exception("Invalid tar file name: " + tarfile + " ... expecting: uima-ducc-<version>-bin.tar.gz") |
| return m.group(1) |
| |
| #----------------------------------------------------------------------------------------- |
| # Create archive directory |
| #----------------------------------------------------------------------------------------- |
| def create_archive(runtime, oversion, nversion): |
| runtimeParent,runtimeName = os.path.split(runtime) |
| now = datetime.datetime.now() |
| nowstr = now.strftime('%Y%m%d-%H%M') |
| archiveParent = os.path.join(runtimeParent, 'ducc_archives') |
| archive = os.path.join(archiveParent, runtimeName + '_' + nowstr) |
| if not os.path.exists(archive): |
| os.makedirs(archive) |
| |
| print " --- Updating DUCC to", nversion, " and archiving", oversion, "to", archive |
| print " NOTE: this update may be reversed by copying back all of the archived files," |
| print " e.g. cp --recursive --remove-destination", archive+"/*", runtime |
| |
| return archive |
| |
| |
| #----------------------------------------------------------------------------------------- |
| # Clear out the old expanded tarball if needed, and expand the new one |
| #----------------------------------------------------------------------------------------- |
| def expand_tarball(tarfile, nversion, outdir): |
| print "\n", " --- Expanding tar file" |
| extract = os.path.join(outdir, 'apache-uima-ducc-' + nversion) |
| try: |
| if ( os.path.exists(extract) ): |
| shutil.rmtree(extract); |
| except: |
| raise Exception("Cannot remove old tarball extract: " + extract) |
| |
| cmd = 'umask 022 && tar -C ' + outdir + ' -xf ' + tarfile |
| rc = os.system(cmd); |
| if ( rc != 0 ): |
| raise Exception("Command fails, rc= " + str(rc) + "\n " + cmd) |
| |
| if not os.path.exists(extract): |
| raise Exception("Cannot extract runtime from tarball " + tarfile + " Expecting to find " + extract) |
| |
| return extract |
| |
| #----------------------------------------------------------------------------------------- |
| # Convert final comments into a DUCC 1.1.0 properties file |
| #----------------------------------------------------------------------------------------- |
| def create_110_properties(source, dest): |
| found = False |
| with open(source) as f: |
| for line in f: |
| if not found: |
| if line.startswith("#=====MARKER====="): |
| found = True |
| outf = open(dest, 'w') |
| else: |
| outf.write('ducc.' + line[1:]) |
| outf.close(); |
| |
| #----------------------------------------------------------------------------------------- |
| # Update directory by archiving all files that are replaced, unless in exclude list |
| # Descend into nested directories if they may contain site-added files to be retained |
| #----------------------------------------------------------------------------------------- |
| def update_directory(olddir, newdir, archdir): |
| |
| global preserveFiles |
| global preserveDirectories |
| global modifiableDirectories |
| global lenRuntime |
| |
| if len(olddir) <= lenRuntime: |
| print "\n", " --- Processing folder:", olddir |
| else: |
| print "\n", " --- Processing folder:", olddir[lenRuntime:] |
| |
| subdirs = [] |
| preserveAll = os.path.basename(newdir) in preserveDirectories |
| if not os.path.exists(archdir): |
| os.mkdir(archdir) |
| files = os.listdir(newdir) |
| for f in files: |
| curf = os.path.join(olddir, f) |
| newf = os.path.join(newdir, f) |
| if f in preserveFiles or preserveAll: |
| if os.path.exists(curf): |
| # If file has changed don't replace it ... if unchanged remove from the extract |
| cmd = 'cmp --quiet ' + newf + ' ' + curf |
| rc = os.system(cmd); |
| if rc != 0: |
| print "Keeping", f, "(new version left in the extract)" |
| else: |
| os.remove(newf) |
| else: |
| print "Adding", f |
| os.rename(newf, curf) |
| else: |
| if os.path.isdir(newf): |
| relf = curf[lenRuntime:] |
| # Save sub-directories that must be processed recursively to preserve any added files |
| if (relf in modifiableDirectories) and os.path.exists(curf): |
| subdirs.append(f) |
| continue |
| if os.path.exists(curf): |
| print "Replacing", f |
| os.rename(curf, os.path.join(archdir,f)) |
| else: |
| print "Adding", f |
| os.rename(newf, curf) |
| # Process directories last to make printout more readable |
| for f in subdirs: |
| curf = os.path.join(olddir, f) |
| newf = os.path.join(newdir, f) |
| update_directory(curf, newf, os.path.join(archdir, f)) |
| # Remove empty directories to make unused files obvious |
| nleft = len(os.listdir(newdir)) |
| if nleft == 0: |
| os.rmdir(newdir) |
| |
| |
| #========================================================================================= |
| # Main program: |
| # tarfile runtime |
| #========================================================================================= |
| |
| |
| #----------------------------------------------------------------------------------------- |
| # Retain a few site-modified files from the archive |
| # - the local_hooks.py may define local versions of 3 utility functions: |
| # find_other_processes, verify_slave_node, verify_master_node |
| # - the keystore is created by ducc-post-install using the password in resources.private |
| # - the site.jsp may define extra ducc-mon buttons |
| # - the ducc.local.js may define local versions of 2 place-holder functions: |
| # ducc_init_local, ducc_update_page_local |
| #----------------------------------------------------------------------------------------- |
| |
| # List of customizable files that must not be replaced |
| preserveFiles = [ 'ducc.classes', 'ducc.administrators', 'ducc.nodes', |
| 'local_hooks.py', 'keystore', 'site.jsp', 'ducc.local.js' ] |
| |
| # List of directories all of whose files nust not be replaced |
| preserveDirectories = [ 'resources.private' ] |
| |
| # List of directories which may contain site-added or modified files |
| # (Their parents must also be included so may also contain other files) |
| modifiableDirectories = [ 'admin', 'lib', |
| 'resources', 'resources/service_monitors', 'resources.private', |
| 'webserver/etc', 'webserver/root', 'webserver/root/$banner', 'webserver/root/js', 'webserver/root/opensources/images', |
| 'webserver', 'webserver/root/opensources' ] |
| |
| if len(sys.argv) != 3: |
| usage() |
| exit(1) |
| |
| rtime = sys.argv[1] |
| tarfile = sys.argv[2] |
| runtime = os.path.realpath(rtime) |
| |
| if not os.path.exists(tarfile): |
| print "ERROR - Missing tar file", tarfile |
| exit(1) |
| nversion = get_nversion(os.path.basename(tarfile)) |
| |
| #----------------------------------------------------------------------------------------- |
| # Check if is a new install ... just untar |
| #----------------------------------------------------------------------------------------- |
| if not os.path.exists(runtime): |
| print " --- A fresh installation of DUCC", nversion, "will be created at", runtime |
| runtimeParent = os.path.dirname(runtime) |
| if not os.path.exists(runtimeParent): |
| os.makedirs(runtimeParent) |
| newducc = expand_tarball( tarfile, nversion, runtimeParent ) |
| os.rename(newducc, runtime) |
| print "\n", " --- Installation completed ... please run './ducc_post_install' from the admin directory" |
| exit(0) |
| |
| #----------------------------------------------------------------------------------------- |
| # Check if invoked from inside the runtime |
| # Probably OK if from admin, but to be safe ... |
| #----------------------------------------------------------------------------------------- |
| |
| curdir = os.path.realpath(os.environ['PWD']) |
| if curdir.startswith(runtime) and curdir != runtime: |
| print "ERROR - Cannot run from inside the runtime" |
| exit(1) |
| |
| #----------------------------------------------------------------------------------------- |
| # Check if appears to be a valid (stopped) DUCC installation |
| #----------------------------------------------------------------------------------------- |
| if not os.path.exists(os.path.join(runtime, 'resources/ducc.properties')): |
| print "ERROR - Not a valid DUCC runtime directory:", runtime |
| exit(1) |
| if os.path.exists(os.path.join(runtime, 'state/ducc.pids')): |
| print "ERROR - DUCC appears to be running ... please run '" + rtime + "/admin/stop_ducc -a'" |
| exit(1) |
| |
| oversion = get_oversion(runtime) |
| if oversion == '1.0.0': |
| print "Sorry, migration not supported for DUCC 1.0.0 at present" |
| exit(9) |
| |
| #----------------------------------------------------------------------------------------- |
| # Create a time-stamped archive and expand the tarball into its parent |
| #----------------------------------------------------------------------------------------- |
| archive = create_archive(runtime, oversion, nversion) |
| newducc = expand_tarball( tarfile, nversion, os.path.dirname(archive) ) |
| |
| #----------------------------------------------------------------------------------------- |
| # May need to create the new webserver directory for 2.0 |
| #----------------------------------------------------------------------------------------- |
| weblogdir = os.path.join(runtime, 'logs/webserver') |
| if not os.path.exists(weblogdir): |
| os.makedirs(weblogdir) |
| |
| #----------------------------------------------------------------------------------------- |
| # Create a site.ducc.properties file if missing ... only for DUCC 1.1.0 |
| # Archive original ducc.properties for safety |
| #----------------------------------------------------------------------------------------- |
| siteProps = os.path.join(runtime, 'resources/site.ducc.properties') |
| if not os.path.exists(siteProps): |
| if oversion != '1.1.0': |
| print "Missing site.ducc.properties - can only be created for 1.1.0" |
| exit(9) |
| currentProps = os.path.join(runtime, 'resources/ducc.properties') |
| archdir = os.path.join(archive,'resources') |
| if not os.path.exists(archdir): |
| os.mkdir(archdir) |
| shutil.copy(currentProps, archdir) |
| |
| originalProps = os.path.join(runtime, 'resources/ducc-1.1.0.properties') |
| create_110_properties(os.path.realpath(sys.argv[0]), originalProps) |
| |
| if not os.path.exists(originalProps): |
| print "ERROR - Failed to create the 1.1.0 properties file from the ending comments in this script" |
| exit(9) |
| |
| # Use the new props manager - use abs fnames as otherwise are relative to the deduced DUCC_HOME/resources |
| cmd = newducc + '/admin/ducc_props_manager --delta ' + originalProps + ' --with ' + currentProps + ' --to ' + siteProps |
| rc = os.system(cmd) |
| if rc != 0: |
| print "ERROR", rc, "Failed to create", siteProps |
| exit(9) |
| print " --- Created a file with just the site-specific properties:", siteProps |
| |
| #----------------------------------------------------------------------------------------- |
| # Add or replace (after archiving) most files and directories found in the new build |
| # EXCEPT for those that are designed to be sire-specific. |
| # Sites may also add files to the 'modifiable' directories and their sub-directories. |
| # The other directories are replaced in toto e.g. 3rd-party ones. |
| #----------------------------------------------------------------------------------------- |
| |
| # Strip the runtime prefix when printing |
| lenRuntime = len(runtime) + 1 |
| update_directory(runtime, newducc, archive) |
| |
| #----------------------------------------------------------------------------------------- |
| # Delete what's left of the extract (just the resources & lib folders) |
| #----------------------------------------------------------------------------------------- |
| print "\n", " --- Files not replaced are left in the extract at", newducc |
| |
| #----------------------------------------------------------------------------------------- |
| # Re-build ducc_ling |
| # Since it needs ducc.properties run the merge from the admin directory |
| #----------------------------------------------------------------------------------------- |
| print "\n", " --- Rebuilding ducc_ling" |
| os.chdir(runtime + '/admin') |
| rc = os.system('./ducc_props_manager --merge ../resources/default.ducc.properties --with ../resources/site.ducc.properties --to ../resources/ducc.properties') |
| if (rc != 0): |
| print "ERROR - failed to create ducc.properties and to rebuild ducc_ling" |
| exit(9) |
| rc = os.system('./build_duccling') |
| if (rc != 0): |
| print "ERROR - failed to rebuild ducc_ling" |
| exit(9) |
| |
| print "\n", " --- Installation completed!" |
| print " NOTE - if your ducc_ling is privileged you should update it" |
| |
| #----------------------------------------------------------------------------------------- |
| # Warn if DB disabled |
| #----------------------------------------------------------------------------------------- |
| siteprops = runtime + '/resources/site.ducc.properties' |
| dbcreated = False |
| with open(siteprops) as f: |
| for line in f: |
| if line.startswith("ducc.database.host"): |
| dbcreated = True |
| break |
| if not dbcreated: |
| print " WARNING - The database has not yet been created - DUCC will have reduced functionality." |
| print " Run db_create then db_loader to convert existing state and history" |
| |
| #->->->->->->-> DO NOT CHANGE ANYTHING BELOW THIS MARKER <-<-<-<-<-<-<- |
| #=====MARKER===== The following are the original ducc.properties shipped with DUCC 1.1.0 |
| #head=<head-node> |
| #jvm=<full-path-to-java-command> |
| #cluster.name=Apache UIMA-DUCC |
| #private.resources=${DUCC_HOME}/resources.private |
| #jms.provider=activemq |
| #broker.protocol=tcp |
| #broker.hostname=${ducc.head} |
| #broker.port=61617 |
| #broker.url.decoration=jms.useCompression=true |
| #broker.name=localhost |
| #broker.jmx.port=1100 |
| #broker.credentials.file=${ducc.private.resources}/ducc-broker-credentials.properties |
| #broker.automanage=true |
| #broker.memory.options=-Xmx1G |
| #broker.configuration=conf/activemq-ducc.xml |
| #broker.home=${DUCC_HOME}/apache-uima/apache-activemq |
| #broker.server.url.decoration=transport.soWriteTimeout=45000 |
| #locale.language=en |
| #locale.country=us |
| #node.min.swap.threshold=0 |
| #admin.endpoint=ducc.admin.channel |
| #admin.endpoint.type=topic |
| #jmx.port=2099 |
| #agent.jvm.args=-Xmx500M |
| #orchestrator.jvm.args=-Xmx1G |
| #rm.jvm.args=-Xmx1G |
| #pm.jvm.args=-Xmx1G |
| #sm.jvm.args=-Xmx1G |
| #db.jvm.args=-Xmx2G |
| #ws.jvm.args=-Xmx2G -Djava.util.Arrays.useLegacyMergeSort=true |
| #environment.propagated=USER HOME LANG |
| #cli.httpclient.sotimeout=0 |
| #signature.required=on |
| #db.configuration.class=org.apache.uima.ducc.db.config.DbComponentConfiguration |
| #db.state.update.endpoint=ducc.db.state |
| #db.state.update.endpoint.type=topic |
| #db.state.publish.rate=15000 |
| #ws.configuration.class=org.apache.uima.ducc.ws.config.WebServerConfiguration |
| #ws.port=42133 |
| #ws.port.ssl=42155 |
| #ws.session.minutes=60 |
| #ws.automatic.cancel.minutes=5 |
| #ws.max.history.entries=4096 |
| #ws.jsp.compilation.directory=/tmp/ducc/jsp |
| #ws.login.enabled=false |
| #ws.visualization.strip.domain=true |
| #jd.configuration.class=org.apache.uima.ducc.jd.config.JobDriverConfiguration |
| #jd.state.update.endpoint=ducc.jd.state |
| #jd.state.update.endpoint.type=topic |
| #jd.state.publish.rate=15000 |
| #jd.queue.prefix=ducc.jd.queue. |
| #jd.queue.timeout.minutes=5 |
| #jd.host.class=JobDriver |
| #jd.host.description=Job Driver |
| #jd.host.memory.size=2GB |
| #jd.host.number.of.machines=1 |
| #jd.host.user=System |
| #jd.share.quantum=400 |
| #threads.limit=500 |
| #driver.jvm.args=-Xmx300M |
| #sm.configuration.class=org.apache.uima.ducc.sm.config.ServiceManagerConfiguration |
| #sm.state.update.endpoint=ducc.sm.state |
| #sm.state.update.endpoint.type=topic |
| #sm.default.monitor.class=org.apache.uima.ducc.cli.UimaAsPing |
| #sm.instance.failure.max=5 |
| #sm.instance.failure.limit=${ducc.sm.instance.failure.max} |
| #sm.instance.failure.window=30 |
| #sm.init.failure.limit=1 |
| #sm.meta.ping.rate=60000 |
| #sm.meta.ping.stability=10 |
| #sm.meta.ping.timeout=15000 |
| #sm.http.port=19989 |
| #sm.http.node=${ducc.head} |
| #sm.default.linger=300000 |
| #orchestrator.configuration.class=org.apache.uima.ducc.orchestrator.config.OrchestratorConfiguration |
| #orchestrator.start.type=warm |
| #orchestrator.state.update.endpoint=ducc.orchestrator.state |
| #orchestrator.state.update.endpoint.type=topic |
| #orchestrator.state.publish.rate=10000 |
| #orchestrator.maintenance.rate=60000 |
| #orchestrator.http.port=19988 |
| #orchestrator.http.node=${ducc.head} |
| #orchestrator.unmanaged.reservations.accepted=true |
| #rm.configuration.class=org.apache.uima.ducc.rm.config.ResourceManagerConfiguration |
| #rm.state.update.endpoint=ducc.rm.state |
| #rm.state.update.endpoint.type=topic |
| #rm.state.publish.rate=10000 |
| #rm.share.quantum=1 |
| #rm.scheduler=org.apache.uima.ducc.rm.scheduler.NodepoolScheduler |
| #rm.class.definitions=ducc.classes |
| #rm.default.memory=4 |
| #rm.init.stability=2 |
| #rm.node.stability=5 |
| #rm.eviction.policy=SHRINK_BY_INVESTMENT |
| #rm.initialization.cap=1 |
| #rm.expand.by.doubling=true |
| #rm.prediction=true |
| #rm.prediction.fudge=120000 |
| #rm.fragmentation.threshold=8 |
| #rm.admin.endpoint=ducc.rm.admin.channel |
| #rm.admin.endpoint.type=queue |
| #agent.configuration.class=org.apache.uima.ducc.agent.config.AgentConfiguration |
| #agent.request.endpoint=ducc.agent |
| #agent.request.endpoint.type=topic |
| #agent.managed.process.state.update.endpoint=ducc.managed.process.state.update |
| #agent.managed.process.state.update.endpoint.type=socket |
| #agent.managed.process.state.update.endpoint.params=transferExchange=true&sync=false |
| #agent.node.metrics.sys.gid.max=500 |
| #agent.node.metrics.publish.rate=30000 |
| #agent.node.metrics.endpoint=ducc.node.metrics |
| #agent.node.metrics.endpoint.type=topic |
| #agent.node.inventory.publish.rate=10000 |
| #agent.node.inventory.publish.rate.skip=30 |
| #agent.node.inventory.endpoint=ducc.node.inventory |
| #agent.node.inventory.endpoint.type=topic |
| #agent.launcher.thread.pool.size=10 |
| #agent.launcher.use.ducc_spawn=true |
| #agent.launcher.ducc_spawn_path=${DUCC_HOME}/admin/ducc_ling |
| #agent.launcher.process.stop.timeout=60000 |
| #agent.launcher.process.init.timeout=7200000 |
| #agent.rogue.process.user.exclusion.filter= |
| #agent.rogue.process.exclusion.filter=sshd:,-bash,-sh,/bin/sh,/bin/bash,grep,ps |
| #agent.share.size.fudge.factor=5 |
| #agent.launcher.cgroups.enable=false |
| #agent.launcher.cgroups.utils.dir=/usr/bin,/bin |
| #agent.exclusion.file=${DUCC_HOME}/resources/exclusion.nodes |
| #pm.configuration.class=org.apache.uima.ducc.pm.config.ProcessManagerConfiguration |
| #pm.request.endpoint=ducc.pm |
| #pm.request.endpoint.type=queue |
| #pm.state.update.endpoint=ducc.pm.state |
| #pm.state.update.endpoint.type=topic |
| #pm.state.publish.rate=15000 |
| #uima-as.configuration.class=org.apache.uima.ducc.agent.deploy.uima.UimaAsServiceConfiguration |
| #uima-as.endpoint=ducc.job.managed.service |
| #uima-as.endpoint.type=socket |
| #uima-as.endpoint.params=transferExchange=true&sync=false |
| #uima-as.saxon.jar.path=file:${DUCC_HOME}/apache-uima/saxon/saxon8.jar |
| #uima-as.dd2spring.xsl.path=${DUCC_HOME}/apache-uima/bin/dd2spring.xsl |
| #flow-controller.specifier=org.apache.uima.ducc.common.uima.DuccJobProcessFC |
