| #!/usr/bin/env python |
| |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # |
| # To use this to execute your Eclipse workspace, install as a symbolic |
| # link in your JRE/bin directory, with any name other than 'viaducc', |
| # for instance, java_viaducc,, linking back to the base version of |
| # viaducc in your ducc_runtime. |
| # |
| # Then configre your Eclipse's launch JRE to use java_viaducc in place |
| # of 'java' |
| |
| import sys |
| import os |
| import re |
| |
| global ducc_mem_size |
| global ducc_mem_size_set |
| global ducc_desc_set |
| global default_jvm |
| global ducc_home |
| |
| ducc_class = 'fixed' |
| java_cmd = None |
| ducc_home = None # tbd in a minute |
| ducc_mem_size = None |
| ducc_mem_size_set = False # whether the user passed a flag to manually set DUCC_MEMORY_SIZE |
| ducc_desc_set = False # whether the user passed a flag to manually set DUCC_DESCRIPTION |
| default_jvm = None |
| |
| # get the default DUCC_HOME right away for message |
| realpath = os.path.realpath(sys.argv[0]) |
| ndx = realpath.rindex('/') |
| ndx = realpath.rindex('/', 0, ndx) |
| ducc_home = realpath[:ndx] |
| |
| def error(message): |
| print "Error:", message |
| sys.exit(1) |
| |
| def usage(): |
| |
| print "Usage: viaducc [defines] [command and parameters]" |
| print " -or-" |
| print " java_viaducc [defines] [java-class and parameters]" |
| print "" |
| print "Where defines include:" |
| print " -DDUCC_MEMORY_SIZE=size-in-gb" |
| print " The default is -DDUCC_MEMORY_SIZE="+str(ducc_mem_size) |
| print"" |
| print " -DDUCC_HOME=alternative-DUCC-runtime" |
| print " The default is -DDUCC_HOME=" + ducc_home |
| print "" |
| print " -DDUCC_CLASS=ducc-scheduling-class" |
| print " The default is -DDUCC_CLASS=" + ducc_class |
| print "" |
| print " -DDUCC_ENVIRONMENT=environment-settings" |
| print " The default is no additional environment. The environment string should be" |
| print ' blank-delimeted and quoted, for example: -DDUCC_ENVIRONMENT="A=B C=D"' |
| if (default_jvm != None ): |
| print "" |
| print " -DJAVA_BIN=specific-java-bin-directory" |
| print " For use with java_viaducc. Details follow." |
| print " The default is -DJAVA_BIN=" + default_jvm |
| print "" |
| print " -DDUCC_DESRIPTION=user-description-string" |
| print " The default description is the program name (viaducc or java_viaducc) with arguments" |
| print "" |
| print " -DDUCC_NO_CANCEL_ON_INTERRUPT" |
| print " To disable process cancellation when the submitting process terminates." |
| print "" |
| print " When invoked as 'viaducc' the command is executed in a DUCC-managed resource" |
| print "with the console (bi-directionally) redirected to the submitter's console." |
| print "" |
| print " When invokded as 'java_viaducc' an appropriate JRE is found and the class" |
| print "is executed as a java class in a DUCC-managed resource, with the console" |
| print "bi-directionally redirected to the submitter's console." |
| print "" |
| print "Notes for java_viaducc:" |
| print "" |
| print " If java_viaducc is installed in a JRE/bin directory, it may be invoked as an" |
| print "alternative to java from Eclipse, allowing Eclipse users to execute their workspaces" |
| print "in a DUCC-managed resource. JAVA_BIN should NOT be used in this case. java_viaducc" |
| print "will use the JRE it is installed in." |
| print "" |
| print " If java_viaducc is used as a 'normal' command, the JRE is searched for in this order:" |
| print " 1. Use the java specified by -DJAVA_BIN=" |
| print " 2. Use the java configured for DUCC" |
| print "" |
| print " If -jar is used it must be the last JVM arg before any command line args" |
| |
| print |
| sys.exit(0) |
| |
| # where should I get java from? |
| def read_properties(): |
| ducc_properties = ducc_home + '/resources/ducc.properties' |
| if ( not os.path.exists(ducc_properties) ): |
| error("Cannot access DUCC_HOME at " + ducc_home) |
| |
| ducc_jvm = None |
| # first see if i'm executed out of some JAVA_HOME somewhere |
| jpath = os.path.abspath(sys.argv[0]) |
| ndx = jpath.rindex('/') |
| jdir = jpath[:ndx] |
| java = jdir + '/java' |
| if (os.path.exists(java)): |
| ducc_jvm = java |
| |
| # nope, we'll use the default ducc java |
| props = open(ducc_properties) |
| try: |
| for line in props: |
| line = line.strip() |
| mobj = re.search('[ =:]+', line) # java props allows apace, =, or : as delimeters |
| if ( mobj ): |
| key = line[:mobj.start()].strip() |
| val = line[mobj.end():].strip() |
| if ( (ducc_jvm is None) and (key == 'ducc.jvm') ): # yes! |
| ducc_jvm = val |
| continue |
| if ( key == 'ducc.rm.share.quantum' ): # yes! |
| ducc_mem = 1 # default quantum may not be the same for all classes |
| continue # and the actual allocation will be rounded up to quantum |
| finally: |
| props.close(); |
| |
| return (ducc_jvm, ducc_mem) |
| |
| def parse_memory_string(text): |
| """Given a string (e.g., "5G", "200m"), Returns the memory requested |
| in bytes. For example: |
| |
| parse_memory_string('500m') should return |
| 500 * 1024 * 1024 |
| parse_memory_string('15G') should return |
| 15 * 1024 * 1024 * 1024 |
| """ |
| |
| # this list should cover us for a while |
| suffixes = { |
| 'k': 1024, |
| 'm': 1024 * 1024, |
| 'g': 1024 * 1024 * 1024, |
| 't': 1024 * 1024 * 1024 * 1024, |
| } |
| |
| text = text.lower() |
| if text[-1] in suffixes: |
| multiplier = suffixes[text[-1]] |
| text = text[:-1] # chop off suffix |
| else: |
| multiplier = 1 |
| |
| return multiplier * int(text) |
| |
| def parse_java_command_line(): |
| """Parse the command line and returns a dictionary including |
| information on the maximum amount of memory the job might need, |
| the Java class name, and its arguments.""" |
| maximum_memory_required = 1024.0 ** 3 # our default if -Xmx not specified |
| args = iter(sys.argv[1:]) |
| skip_next_arg = False |
| |
| # scan args until we find the Java class |
| while 1: |
| arg = next(args).lower() |
| if skip_next_arg: |
| skip_next_arg = False |
| continue |
| |
| if arg.startswith('-'): |
| # these are the only options that take an argument and require |
| # a space between the flag and the argument |
| if arg in ('-cp', '-classpath'): |
| skip_next_arg = True |
| elif arg.startswith('-xmx'): # maximum Java heap size |
| maximum_memory_required = parse_memory_string(arg[4:]) |
| continue |
| |
| break |
| |
| return dict(maximum_memory_required=maximum_memory_required, |
| java_class_name=arg, java_class_args=list(args)) |
| |
| def main(): |
| |
| global ducc_mem_size |
| global ducc_mem_size_set |
| global ducc_desc_set |
| global default_jvm |
| global ducc_home |
| |
| if ( len(sys.argv) == 1 ): |
| usage() |
| |
| ducc_class = 'fixed' |
| java_cmd = None |
| ducc_env = '' |
| enable_cancel = True |
| ducc_desc = os.path.basename(sys.argv[0]) |
| |
| # remember to add the '=' at the end if following value |
| p_mem_size = '-DDUCC_MEMORY_SIZE=' |
| p_class = '-DDUCC_CLASS=' |
| p_jvm_dir = '-DJAVA_BIN=' |
| p_ducc_home = '-DDUCC_HOME=' |
| p_env = '-DDUCC_ENVIRONMENT=' |
| p_no_int = '-DDUCC_NO_CANCEL_ON_INTERRUPT' |
| p_desc = '-DDUCC_DESCRIPTION=' |
| |
| need_java = False |
| if ( os.path.basename(sys.argv[0]) != 'viaducc' ): |
| # if invokded as 'java_viaducc we must inject some kind of jvm as the executable |
| # this is more liberal: any link other than 'viaducc' caused injection of the |
| # jvm, to allow multiple, different symlinks (in case there are multiple, |
| # different versions of ducc, not really reccomended, but supported) |
| need_java = True |
| |
| ducc_args = [] |
| for arg in sys.argv[1:]: |
| |
| if (arg in ('-h', '-?', '-help', '--help') ): |
| # have to wait for parsing the args to look up DUCC_HOME so we defer emitting the message |
| usage() |
| |
| elif (arg.startswith(p_mem_size) ): |
| ducc_mem_size = arg[len(p_mem_size):] |
| ducc_mem_size_set = True |
| |
| elif (arg.startswith(p_class) ): |
| ducc_class = arg[len(p_class):] |
| |
| elif (arg.startswith(p_jvm_dir) ): |
| java_cmd = arg[len(p_jvm_dir):] + '/java' |
| |
| elif (arg.startswith(p_ducc_home) ): |
| ducc_home = arg[len(p_ducc_home):] |
| |
| elif (arg.startswith(p_env) ): |
| ducc_env = ' --environment "' + arg[len(p_env):] + '"' |
| |
| elif (arg.startswith(p_desc) ): |
| ducc_desc = arg[len(p_desc):] |
| ducc_desc_set = True |
| |
| elif (arg.startswith(p_no_int) ): |
| enable_cancel = False |
| |
| else: |
| ducc_args.append("'" + arg + "'") |
| |
| p = read_properties() |
| default_jvm = p[0] |
| if (ducc_mem_size is None): |
| ducc_mem_size = p[1] |
| |
| description = os.path.basename(sys.argv[0]) |
| if ( need_java and (java_cmd is None) ): |
| java_cmd = default_jvm |
| if ( (java_cmd is None) or (java_cmd == '') ): |
| error("Cannot figure out where java is") |
| |
| if need_java: |
| java_command_line_info = parse_java_command_line() |
| description += ' %s %s' % (java_command_line_info['java_class_name'], |
| ' '.join(java_command_line_info['java_class_args'])) |
| |
| import math |
| # convert to gigabytes, round up |
| maximum_memory_required = java_command_line_info['maximum_memory_required'] / (1024.0 ** 3) |
| maximum_memory_required = int(math.ceil(maximum_memory_required)) |
| if maximum_memory_required > float(ducc_mem_size): |
| if ducc_mem_size_set: |
| print ("Warning: DUCC memory size specified as %sg but max heap set to %sg.\n" |
| " Consider increasing -DDUCC_MEMORY_SIZE=<size>\n") % \ |
| (ducc_mem_size, maximum_memory_required) |
| else: |
| print ("Info: DUCC memory size is not specified but asking JVM to set max heap to %sg.\n" |
| " Setting DUCC memory size to %sg. Use -DDUCC_MEMORY_SIZE=<size> to specify\n") % \ |
| (maximum_memory_required, maximum_memory_required) |
| ducc_mem_size = maximum_memory_required |
| |
| # DUCC_HOME isn't ususally needed but viaducc is slithery and may end up tryng to |
| # execute from somebody else's ducc so let's be sure we point at the right one |
| os.environ['DUCC_HOME'] = ducc_home |
| CMD = ducc_home + '/bin/ducc_process_submit' |
| CMD = CMD + ' --attach_console' |
| CMD = CMD + ' --process_memory_size ' + str(ducc_mem_size) |
| CMD = CMD + ' --scheduling_class ' + ducc_class |
| if ducc_desc_set: |
| CMD = CMD + ' --description %r' % ducc_desc |
| else: |
| if need_java: |
| CMD = CMD + ' --description %r' % description # %r so it will be quoted |
| else: |
| CMD = CMD + ' --description viaducc' |
| |
| CMD = CMD + ' --wait_for_completion' |
| if (enable_cancel): |
| CMD = CMD + ' --cancel_on_interrupt' |
| CMD = CMD + ' --working_directory ' + os.getcwd() |
| CMD = CMD + ducc_env |
| |
| if ( need_java ): |
| CMD = CMD + ' --process_executable ' + java_cmd |
| if ( len(ducc_args) > 0): |
| CMD = CMD + ' --process_executable_args "' + ' '.join(ducc_args) + '"' |
| else: |
| if ( len(ducc_args) == 0 ): |
| error("No command specified.") |
| |
| CMD = CMD + ' --process_executable ' + ducc_args[0] |
| if ( len(ducc_args[1:]) > 0): |
| CMD = CMD + ' --process_executable_args "' + ' '.join(ducc_args[1:]) + '"' |
| |
| print CMD |
| sys.stdout.flush() # UIMA-4168 |
| |
| rc = os.system(CMD) |
| sys.exit(rc); |
| |
| main() |
