| #!/usr/bin/env python |
| # ----------------------------------------------------------------------- |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # ----------------------------------------------------------------------- |
| |
| |
| import os |
| import sys |
| |
| from ducc_boot import * |
| set_ducc_home() |
| |
| import string |
| import re |
| import getopt |
| |
| global errors |
| errors = 0 |
| |
| from ducc_util import DuccUtil |
| from ducc_util import DuccProperties |
| from ducc import Ducc |
| |
| class VerifyDucc(DuccUtil): |
| |
| # |
| # Make sure ducc_ling is set up ok. |
| # Make sure ducc_ling is configured in ducc.properties |
| # Make sure the broker address is set in ducc.properties. |
| # Make sure reserved.nodes has at least one of the nodes named in ducc.nodes |
| # |
| |
| def check_duckling(self): |
| global errors |
| self.verify_duccling() |
| |
| # |
| # allnodes is a dictionary. The key is the name of a nodefile, the value is a |
| # list of all nodes in that file. |
| def check_remote_nodes(self, allnodes, environment): |
| global errors |
| |
| duckling_ok = True |
| |
| for (nodefile, nodes) in allnodes.items(): |
| for node in nodes: |
| checklist = [] |
| mem = '' |
| lines = self.ssh(node, True, self.DUCC_HOME + '/admin/verify_ducc', '-d') |
| for line in lines: |
| line = line.strip() |
| if ( line.startswith('ENV') ): |
| if (line.find('not found') >= 0): |
| print 'NOTOK', node, 'from', nodefile, line |
| errors = errors + 1 |
| else: |
| checklist.append(line) |
| elif (line.startswith('MEM')): |
| mem = line |
| elif ( line != 'ducc_ling OK'): |
| print 'NOTOK:', node, 'from', nodefile, line, mem |
| errors = errors + 1 |
| duccling_ok = False |
| else: |
| print 'OK:', node, line, mem |
| for ( local, remote ) in zip(environment, checklist): |
| if (local != remote): |
| print 'DIFFERENCE on node', node, ':' |
| print '%20s %s' % ('LOCAL:', local) |
| print '%20s %s' % (node, remote) |
| |
| return duckling_ok |
| |
| |
| def check_broker(self, broker_conf): |
| global errors |
| |
| #broker_conf = broker_loc + '/apache-activemq-5.5.0/conf/activemq.xml' |
| broker_uri = None |
| |
| #look for something like this: <transportConnector name="openwire" uri="tcp://0.0.0.0:61616"/> |
| f = open(broker_conf) |
| for line in f: |
| if ( not line ): |
| break |
| line = line.strip() |
| toks = line.split() |
| if ( not line ): |
| continue |
| if ( toks[0] == '<transportConnector' ): |
| # we need to parse out the uri and if localhost or 0.0.0.0 is specified, |
| # we need to replace it with the hostname |
| |
| url = toks[2] |
| toks = url.split('"') |
| address = toks[1] |
| toks = address.split(':') |
| protocol = toks[0] |
| host = toks[1] |
| if ( (toks[1] == '//0.0.0.0') or (toks[1] == 'localhost') ): |
| host = os.uname()[1] |
| else: |
| host = host[2:] # strip // |
| shorthost = host |
| ttoks = host.split('.') #strip domain for easier matching later |
| if ( len(ttoks) > 1 ): |
| shorthost = ttoks[0] |
| port = toks[2] |
| |
| amqok = True |
| protocols = ['tcp', 'nio'] |
| if ( not ((protocol in protocols) and (self.broker_protocol in protocols)) ): |
| amqok = False |
| if ( host != self.broker_host ): |
| amqok = False |
| if ( port != self.broker_port ): |
| amqok = False |
| |
| if ( amqok ): |
| print 'OK DUCC Configured ActiveMQ URL', self.broker_url, 'matches ActiveMQ configured URL.' |
| else: |
| print 'NOTOK DUCC Configured ActiveMQ URL', self.broker_url, 'does not match ActiveMQ configured URL:', address |
| errors = errors + 1 |
| |
| return amqok |
| |
| def compare_nodes(self, n1, n2): |
| |
| if ( n1 == n2 ): # exact match - covers both short and both long |
| return True |
| |
| if ( n1.find('.') >= 0 ): # shortened n1 == n2? |
| t1 = n1.split('.') |
| n1A = t1[0] |
| if ( n1A == n2 ): |
| return True |
| |
| if ( n2.find('.') >= 0 ): # n1 == shortened n2? |
| t2 = n2.split('.') |
| n2A = t2[0] |
| if ( n1 == n2A ): |
| return True |
| |
| return False |
| |
| def check_nodepools(self, allnodes): |
| global errors |
| |
| classfile = self.ducc_properties.get('ducc.rm.class.definitions') |
| classfile = self.resolve(classfile, self.propsfile) # resolve the classfile relative to ducc.properties |
| |
| classprops = DuccProperties() |
| classprops.load(classfile) |
| |
| # |
| # First make sure that all the nodepools that are declared have definition files |
| # and that the defined nodes are in some nodelist. |
| # |
| nodepools = classprops.get('scheduling.nodepool').split() |
| nodepools_ok = True |
| for np in nodepools: |
| npkey = 'scheduling.nodepool.' + np |
| npfilename = classprops.get(npkey) |
| npfile = self.DUCC_HOME + '/resources/' + npfilename |
| if ( not os.path.exists(npfile) ): |
| print 'NOTOK: Cannot find nodepool file', npfile |
| errors = errors + 1 |
| continue |
| |
| npnodes = DuccProperties() |
| npnodes.load(npfile) |
| for ( node, v ) in npnodes.items(): |
| found = False |
| for (nodefile, nodelist) in allnodes.items(): |
| for n in nodelist: |
| if ( self.compare_nodes(n, node)): |
| found = True |
| break |
| if ( not found ): |
| print 'NOTOK: Cannot find node defined in pool "' +np+'" in any nodefile:', node |
| errors = errors + 1 |
| nodepools_ok = False |
| |
| # |
| # Now make sure that all classes that reference nodepools have corresponding |
| # nodepool definitions |
| # |
| |
| for ( k, v ) in classprops.items(): |
| if ( k.startswith('scheduling.class.') and k.endswith('.nodepool') ): |
| if ( not ( v in nodepools ) ): |
| toks = k.split('.') |
| classname = toks[2] |
| print 'NOTOK: Class', classname, 'references non-existent nodepool', v |
| errors = errors + 1 |
| nodepools_ok = False |
| |
| if ( nodepools_ok ): |
| print 'OK: All nodepools are verified' |
| else: |
| print 'NOTOK: some nodepools are not correctly defined.' |
| |
| def usage(self, msg=None): |
| if ( msg != None ): |
| print msg |
| print 'verify_ducc [-b broker_install_dir] [-d] [-n] [-h | -?] [-v]' |
| print '' |
| print 'Where' |
| print ' -b broker_conf - specify the configuration file for the broker you are using.' |
| print ' Example: verify_ducc -b /home/challngr/amqbroker/amq/conf/activemq-nojournal5.xml' |
| print '' |
| print ' -n nodelist - use this nodelist.' |
| print ' May be specified multiple times.' |
| print '' |
| print ' -h or -? gives this help message' |
| print '' |
| print ' -v shows the DUCC version' |
| print '' |
| print 'Internally verify_ducc recursively uses a -d flag, not intended for general use.' |
| print ' -d just verify duccling on the local host' |
| print '' |
| print 'When invoked with no arguments, this performs installation verification checks for DUCC.' |
| sys.exit(1) |
| |
| def main(self, argv): |
| global errors |
| |
| environ = self.show_ducc_environment() |
| jars = [] |
| for e in environ: |
| if ( e.startswith('ENV:') ): |
| jars.append(e) |
| print e |
| |
| do_duckling = False |
| nodelists = [] |
| amqhome = None |
| |
| try: |
| opts, args = getopt.getopt(argv, 'b:dn:h?v') |
| except: |
| self.usage('Invalid arguments.') |
| |
| |
| for ( o, a ) in opts: |
| if ( o == '-v' ) : |
| print self.version() |
| sys.exit(0) |
| elif ( o == '-d'): |
| do_duckling = True |
| elif ( o == '-b'): |
| amqhome = a |
| elif ( o == '-n'): |
| nodelists.append(a) |
| elif ( o == '-h'): |
| self.usage(None) |
| elif ( o == '-?'): |
| self.usage(None) |
| else: |
| self.usage('bad arg: ' + a) |
| |
| if ( do_duckling ): |
| self.check_duckling() |
| return |
| |
| HOME = os.environ['HOME'] |
| DEFAULT_MQ = HOME + '/activemq/apache-activemq-5.5.0/conf/activemq.xml' |
| MQ_DIR = DEFAULT_MQ |
| MQ_VERSION = 'apache-activemq-5.5.0' # required |
| |
| if ( amqhome == None ): |
| MQ_DIR = DEFAULT_MQ |
| else: |
| MQ_DIR = amqhome |
| print 'ActiveMQ directory:', MQ_DIR |
| |
| if ( len(nodelists) == 0 ): |
| nodelists = self.default_nodefiles |
| |
| activemq_loc = MQ_DIR |
| |
| # read all nodes into a dictionary keyed off the nodefile name |
| tmp = {} |
| for nodefile in nodelists: |
| tmp = self.read_nodefile(nodefile, tmp) |
| |
| # transfer nodes into an array with only stuff that could be |
| # found, emitting errors as needed |
| allnodes = {} |
| for ( k, v ) in tmp.items(): |
| if ( v == None ): |
| print 'NOTOK: Cannot read nodefile', k |
| errors = errors + 1 |
| else: |
| allnodes[k] = v |
| |
| self.check_broker(activemq_loc) # reads activemq.conf and verifies the url against the configured url |
| self.check_nodepools(allnodes) |
| self.check_remote_nodes(allnodes, jars) |
| |
| print 'Verified with', errors, 'errors' |
| |
| |
| if __name__ == "__main__": |
| verifier = VerifyDucc() |
| verifier.main(sys.argv[1:]) |
| |
| |
