blob: 909e419c18010e2a32332a2984e4f190c894f33f [file] [log] [blame]
#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ConfigParser import SafeConfigParser
from subprocess import *
from datetime import datetime
import time
import os
import logging
import json
from os import sys, path
from health_checks.utility import getHealthChecksData
class StatusCodes:
SUCCESS = 0
FAILED = 1
INVALID_INP = 2
RUNNING = 3
STOPPED = 4
STARTING = 5
class Log:
INFO = 'INFO'
ALERT = 'ALERT'
CRIT = 'CRIT'
NOTIF = 'NOTIF'
class Config:
SLEEP_SEC = 1
RETRY_ITERATIONS = 10
RETRY_FOR_RESTART = 5
MONITOR_LOG = '/var/log/monitor.log'
HEALTH_CHECKS_DIR = 'health_checks'
MONITOR_RESULT_FILE_SUFFIX = 'monitor_results.json'
FAILING_CHECKS_FILE = 'failing_health_checks'
def getServicesConfig( config_file_path = "/etc/monitor.conf" ):
"""
Reads the process configuration from the config file.
Config file contains the processes to be monitored.
"""
process_dict = {}
parser = SafeConfigParser()
parser.read( config_file_path )
for section in parser.sections():
process_dict[section] = {}
for name, value in parser.items(section):
process_dict[section][name] = value
printd (" %s = %r" % (name, value))
return process_dict
def printd (msg):
"""
prints the debug messages
"""
#for debug
#print msg
f= open(Config.MONITOR_LOG, 'w' if not path.isfile(Config.MONITOR_LOG) else 'r+')
f.seek(0, 2)
f.write(str(msg)+"\n")
f.close()
print str(msg)
def raisealert(severity, msg, process_name=None):
""" Writes the alert message"""
#timeStr=str(time.ctime())
if process_name is not None:
log = '['+severity +']'+" " + '['+process_name+']' + " " + msg +"\n"
else:
log = '['+severity+']' + " " + msg +"\n"
logging.basicConfig(level=logging.INFO,filename='/var/log/routerServiceMonitor.log',format='%(asctime)s %(message)s')
logging.info(log)
msg = 'logger -t monit '+ log
pout = Popen(msg, shell=True, stdout=PIPE)
print "[Alert] " + msg
def isPidMatchPidFile(pidfile, pids):
""" Compares the running process pid with the pid in pid file.
If a process with multiple pids then it matches with pid file
"""
if pids is None or isinstance(pids,list) != True or len(pids) == 0:
printd ("Invalid Arguments")
return StatusCodes.FAILED
if not path.isfile(pidfile):
#It seems there is no pid file for this service
printd("The pid file "+pidfile+" is not there for this process")
return StatusCodes.FAILED
fd=None
try:
fd = open(pidfile,'r')
except:
printd("pid file: "+ pidfile +" open failed")
return StatusCodes.FAILED
inp = fd.read()
if not inp:
fd.close()
return StatusCodes.FAILED
printd("file content of pidfile " + pidfile + " = " + str(inp).strip())
printd(pids)
tocheck_pid = inp.strip()
for item in pids:
if str(tocheck_pid) == item.strip():
printd("pid file matched")
fd.close()
return StatusCodes.SUCCESS
fd.close()
return StatusCodes.FAILED
def checkProcessRunningStatus(process_name, pidFile):
printd("checking the process " + process_name)
cmd = ''
pids = []
cmd = 'pidof ' + process_name
printd(cmd)
#cmd = 'service ' + process_name + ' status'
pout = Popen(cmd, shell=True, stdout=PIPE)
exitStatus = pout.wait()
temp_out = pout.communicate()[0]
#check there is only one pid or not
if exitStatus == 0:
pids = temp_out.strip().split(' ')
printd("pid(s) of process %s are %s " %(process_name, pids))
#there is more than one process so match the pid file
#if not matched set pidFileMatched=False
printd("Checking pid file")
if isPidMatchPidFile(pidFile, pids) == StatusCodes.SUCCESS:
return True,pids
printd("pid of exit status %s" %exitStatus)
return False,pids
def restartService(service_name):
cmd = 'service ' + service_name + ' restart'
cout = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
return_val = cout.wait()
if return_val == 0:
printd("The service " + service_name +" recovered successfully ")
msg="The process " +service_name+" is recovered successfully "
raisealert(Log.INFO,msg,service_name)
return True
else:
printd("process restart failed ....")
return False
def checkProcessStatus( process ):
"""
Check the process running status, if not running tries to restart
Returns the process status and if it was restarted
"""
process_name = process.get('processname')
service_name = process.get('servicename')
pidfile = process.get('pidfile')
#temp_out = None
restartFailed=False
pidFileMatched=False
pids=''
cmd=''
if process_name is None:
printd ("\n Invalid Process Name")
return StatusCodes.INVALID_INP, False
status, pids = checkProcessRunningStatus(process_name, pidfile)
if status == True:
printd("The process is running ....")
return StatusCodes.RUNNING, False
else:
printd("Process %s is not running trying to recover" %process_name)
#Retry the process state for few seconds
for i in range(1, Config.RETRY_ITERATIONS):
time.sleep(Config.SLEEP_SEC)
if i < Config.RETRY_FOR_RESTART: # this is just for trying few more times
status, pids = checkProcessRunningStatus(process_name, pidfile)
if status == True:
raisealert(Log.ALERT, "The process detected as running", process_name)
break
else:
printd("Process %s is not running checking the status again..." %process_name)
continue
else:
msg="The process " +process_name+" is not running trying recover "
raisealert(Log.INFO,process_name,msg)
if service_name == 'apache2':
# Killing apache2 process with this the main service will not start
for pid in pids:
cmd = 'kill -9 '+pid
printd(cmd)
Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
if restartService(service_name) == True:
break
else:
restartFailed = True
continue
#for end here
if restartFailed == True:
msg="The process %s recover failed "%process_name
raisealert(Log.ALERT,process_name,msg)
printd("Restart failed after number of retries")
return StatusCodes.STOPPED, False
return StatusCodes.RUNNING, True
def monitProcess( processes_info ):
"""
Monitors the processes which got from the config file
"""
checkStartTime = time.time()
service_status = {}
failing_services = []
if len( processes_info ) == 0:
printd("No config items provided - means a redundant VR or a VPC Router")
return service_status, failing_services
print "[Process Info] " + json.dumps(processes_info)
#time for noting process down time
csec = repr(time.time()).split('.')[0]
for process,properties in processes_info.items():
printd ("---------------------------\nchecking the service %s\n---------------------------- " %process)
serviceName = process + ".service"
processStatus, wasRestarted = checkProcessStatus(properties)
if processStatus != StatusCodes.RUNNING:
printd( "\n Service %s is not Running"%process)
checkEndTime = time.time()
service_status[serviceName] = {
"success": "false",
"lastUpdate": str(int(checkStartTime * 1000)),
"lastRunDuration": str((checkEndTime - checkStartTime) * 1000),
"message": "service down at last check " + str(csec)
}
failing_services.append(serviceName)
else:
checkEndTime = time.time()
service_status[serviceName] = {
"success": "true",
"lastUpdate": str(int(checkStartTime * 1000)),
"lastRunDuration": str((checkEndTime - checkStartTime) * 1000),
"message": "service is running" + (", was restarted" if wasRestarted else "")
}
return service_status, failing_services
def execute(script, checkType = "basic"):
checkStartTime = time.time()
cmd = "./" + script + " " + checkType
printd ("Executing health check script command: " + cmd)
pout = Popen(cmd, shell=True, stdout=PIPE)
exitStatus = pout.wait()
output = pout.communicate()[0].strip()
checkEndTime = time.time()
if exitStatus == 0:
if len(output) > 0:
printd("Successful execution of " + script)
return {
"success": "true",
"lastUpdate": str(int(checkStartTime * 1000)),
"lastRunDuration": str((checkEndTime - checkStartTime) * 1000),
"message": output
}
return {} #Skip script if no output is received
else:
printd("Script execution failed " + script)
return {
"success": "false",
"lastUpdate": str(int(checkStartTime * 1000)),
"lastRunDuration": str((checkEndTime - checkStartTime) * 1000),
"message": output
}
def main(checkType = "basic"):
startTime = time.time()
'''
Step1 : Get Services Config
'''
printd("monitoring started")
configDict = getServicesConfig()
'''
Step2: Monitor services and Raise Alerts
'''
monitResult = {}
failingChecks = []
if checkType == "basic":
monitResult, failingChecks = monitProcess(configDict)
'''
Step3: Run health check scripts as needed
'''
hc_data = getHealthChecksData()
if hc_data is not None and "health_checks_enabled" in hc_data and hc_data['health_checks_enabled']:
hc_exclude = hc_data["excluded_health_checks"] if "excluded_health_checks" in hc_data else []
for f in os.listdir(Config.HEALTH_CHECKS_DIR):
if f in hc_exclude:
continue
fpath = path.join(Config.HEALTH_CHECKS_DIR, f)
if path.isfile(fpath) and os.access(fpath, os.X_OK):
ret = execute(fpath, checkType)
if len(ret) == 0:
continue
if "success" in ret and ret["success"].lower() == "false":
failingChecks.append(f)
monitResult[f] = ret
'''
Step4: Write results to the json file for admins/management server to read
'''
endTime = time.time()
monitResult["lastRun"] = {
"start": str(datetime.fromtimestamp(startTime)),
"end": str(datetime.fromtimestamp(endTime)),
"duration": str(endTime - startTime)
}
with open(checkType + "_" + Config.MONITOR_RESULT_FILE_SUFFIX, 'w') as f:
json.dump(monitResult, f, ensure_ascii=False)
failChecksFile = checkType + "_" + Config.FAILING_CHECKS_FILE
if len(failingChecks) > 0:
fcs = ""
for fc in failingChecks:
fcs = fcs + fc + ","
fcs = fcs[0:-1]
with open(failChecksFile, 'w') as f:
f.write(fcs)
elif path.isfile(failChecksFile):
os.remove(failChecksFile)
if __name__ == "__main__":
checkType = "basic"
if len(sys.argv) == 2:
if sys.argv[1] == "advanced":
main("advanced")
elif sys.argv[1] == "basic":
main("basic")
else:
printd("Error: Unknown type of test: " + sys.argv)
else:
main("basic")
main("advanced")