| #!/usr/bin/python |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| # TODO: |
| # add multipath -ll |
| # add iscsiadm output |
| # |
| import time |
| import socket |
| import getopt |
| import sys |
| import subprocess, threading |
| import logging |
| import logging.handlers |
| import re |
| import shutil |
| import os |
| |
| """ a class to do checks with as a thread so we can have nice timeouts """ |
| class Check(object): |
| def __init__(self, cmd="", failcmd="", primary="", |
| file="", timeout="120", interval=1, logger="", |
| check=False): |
| self.file=file |
| self.cmd=cmd |
| self.failcmd=failcmd |
| self.primary=primary |
| self.timeout=timeout |
| self.interval=interval |
| self.process=None |
| self.logger=logger |
| self.check=check |
| self.ok=None |
| self.results={} |
| |
| def readhb(self,file=""): |
| if os.path.isfile(file): |
| text_file = open("%s" % file, "r") |
| line=text_file.readline() |
| text_file.close() |
| return line |
| return 0 |
| |
| def writehb(self,file=""): |
| if file: |
| nfile="%s.new" % (file) |
| epoch=time.time() |
| text_file = open("%s" % nfile, "w") |
| text_file.write("%s" % epoch) |
| text_file.close() |
| shutil.move(nfile,file) |
| self.logger.debug('Worked on file %s for %s' % |
| (file, (time.time() - epoch))) |
| |
| """ We only want mounted nfs filesystems """ |
| def nfsoutput(self): |
| command="mount -v -t nfs" |
| p=subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) |
| lines=map(lambda line: line.split()[2], p.stdout.readlines()) |
| test=re.compile("^%s" % (primary)) |
| lines=filter(test.search, lines) |
| return lines |
| |
| """ |
| The main run for all checks we do, |
| everything is in here on purpose. |
| |
| the other FSs to heartbeat should be added to filesystems...! |
| """ |
| def run(self, timeout): |
| def target(): |
| filesystems=[] |
| filesystems.extend(self.nfsoutput()) |
| for fs in filesystems: |
| if self.file: |
| if self.check==False: |
| self.writehb("%s/%s" % (fs,file)) |
| else: |
| res=self.readhb("%s/%s" % (fs,file)) |
| delay = time.time() - float(res) |
| if (delay < timeout) and self.ok == None: |
| self.logger.info("%s/%s is ok %s with %s" % (fs,file,timeout,delay)) |
| self.ok = True |
| elif (delay > timeout): |
| self.logger.warning("%s/%s exceeded timeout %s with %s" % (fs,file,timeout, delay)) |
| self.ok = False |
| self.results[fs] = [self.ok, delay] |
| |
| epoch=time.time() |
| if self.cmd: |
| self.logger.debug('Executing: %s' % (cmd)) |
| self.process = subprocess.Popen(self.cmd, shell=True) |
| self.process.communicate() |
| self.logger.info('Executed: %s in %s' % |
| (cmd, (time.time() - epoch))) |
| |
| thread = threading.Thread(target=target) |
| thread.start() |
| thread.join(self.timeout) |
| if thread.isAlive() and self.check == False: |
| self.logger.critical('Critical: thread timeout; %s' % (timeout)) |
| if self.failcmd: |
| self.logger.critical('Critical: executing; %s' % (failcmd)) |
| p=subprocess.Popen(failcmd, shell=True, stdout=subprocess.PIPE) |
| |
| """ here we figure out what we're running on more or less """ |
| def figureOutPrimary(): |
| redhat="/etc/redhat-release" |
| if os.path.isfile(redhat): |
| for line in open(redhat): |
| if "XenServer" in line: |
| return "/var/run/sr-mount" |
| if "Oracle VM server" in line: |
| return "/OVS/Repositories/" |
| print "Unknown hypervisor, consider adding it, exiting" |
| sys.exit(42) |
| |
| """ The logger is here """ |
| def Logger(level=logging.DEBUG): |
| logger = logging.getLogger('cs-heartbeat') |
| logger.setLevel(level) |
| handler = logging.handlers.SysLogHandler(address = '/dev/log') |
| logger.addHandler(handler) |
| return logger |
| |
| """ main for preso-dent """ |
| if __name__ == '__main__': |
| me=os.path.basename(__file__) |
| timeout=120 |
| interval=1 |
| hostname=socket.gethostname() |
| file=".hb-%s" % (hostname) |
| cmd="" |
| level=logging.DEBUG |
| primary="" |
| checkstate=False |
| failcmd=("echo 1 > /proc/sys/kernel/sysrq " |
| "&& " |
| "echo c > /proc/sysrq-trigger") |
| |
| # xenserver: |
| if me == "heartbeat": |
| # String result = callHostPluginPremium(conn, "heartbeat", |
| # "host", _host.uuid, |
| # "timeout", Integer.toString(_heartbeatTimeout), |
| # "interval", Integer.toString(_heartbeatInterval)); |
| # if (result == null || !result.contains("> DONE <")) { |
| try: |
| opts, args = getopt.getopt(sys.argv[1:], "h:y:i:s", |
| [ 'host', 'timeout', 'interval', 'state']) |
| except getopt.GetoptError: |
| print """Usage: |
| host: host guid. |
| timeout: timeout to fail on |
| interval: time between checks |
| state: check the state""" |
| sys.exit() |
| for o, a in opts: |
| if o in ('host'): |
| file="hb-%s" % (a) |
| if o in ('timeout'): |
| timeout=a |
| if o in ('interval'): |
| interval=a |
| if o in ('state'): |
| checkstate=True |
| # OVM3: |
| else: |
| # get options |
| try: |
| opts, args = getopt.getopt(sys.argv[1:], "g:p:f:c:t:i:s", |
| [ 'guid=', 'primary=','failcmd=','cmd=','timeout=','interval', 'state']) |
| except getopt.GetoptError: |
| print """Usage: |
| --guid|-g: guid of the host to check |
| --primary|-p: match for primary storage to monitor. |
| --failcmd|-f: executed on timeout. |
| --cmd|-c: command to execute next to hb file(s) on primary. |
| --timeout|-t: excute failcmd after timeout(s) is hit. |
| --interval|-i: run the checks every %ss> |
| --state|-s check state""" |
| sys.exit() |
| |
| for o, a in opts: |
| if o in ('-g', '--guid'): |
| file=".hb-%s" % (a) |
| if o in ('-p', '--primary'): |
| primary=a |
| if o in ('-f', '--failcmd'): |
| failcmd=a |
| if o in ('-c', '--cmd'): |
| cmd=a |
| if o in ('-t', '--timeout'): |
| timeout=int(a) |
| if o in ('-i', '--interval'): |
| interval=int(a) |
| if o in ('-s', '--state'): |
| checkstate=True |
| |
| if primary == "": |
| primary=figureOutPrimary() |
| |
| logger=Logger(level=level) |
| if checkstate == False: |
| os.chdir("/") |
| # os.setsid() |
| os.umask(0) |
| try: |
| pid = os.fork() |
| if pid > 0: |
| # exit first parent |
| if me == "heartbeat": |
| print "> DONE <" |
| sys.exit(0) |
| except OSError, e: |
| print >>sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror) |
| sys.exit(1) |
| |
| checker=Check(cmd=cmd, |
| failcmd=failcmd, |
| file=file, |
| timeout=timeout, |
| interval=interval, |
| logger=logger, |
| check=checkstate); |
| |
| while True: |
| start=time.time() |
| checker.run(timeout) |
| runtime=time.time() - start |
| logger.debug("cmd time: %s" % (runtime)) |
| if checkstate: |
| for fs in checker.results: |
| print "%s: %s" % (fs, checker.results[fs]) |
| if checker.ok == False: |
| sys.exit(1) |
| else: |
| sys.exit(0) |
| if runtime > interval: |
| logger.warning('Warning: runtime %s bigger than interval %s' % |
| (runtime, interval)) |
| else: |
| time.sleep(interval) |
