| // Licensed to the Apache Software Foundation (ASF) under one | |
| // or more contributor license agreements. See the NOTICE file | |
| // distributed with this work for additional information | |
| // regarding copyright ownership. The ASF licenses this file | |
| // to you under the Apache License, Version 2.0 (the | |
| // "License"); you may not use this file except in compliance | |
| // with the License. You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, | |
| // software distributed under the License is distributed on an | |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| // KIND, either express or implied. See the License for the | |
| // specific language governing permissions and limitations | |
| // under the License. | |
| package com.cloud.alert; | |
| import java.io.UnsupportedEncodingException; | |
| import java.text.DecimalFormat; | |
| import java.util.ArrayList; | |
| import java.util.Date; | |
| import java.util.HashMap; | |
| import java.util.List; | |
| import java.util.Map; | |
| import java.util.Properties; | |
| import java.util.Timer; | |
| import java.util.TimerTask; | |
| import javax.ejb.Local; | |
| import javax.inject.Inject; | |
| import javax.mail.Authenticator; | |
| import javax.mail.Message.RecipientType; | |
| import javax.mail.MessagingException; | |
| import javax.mail.PasswordAuthentication; | |
| import javax.mail.Session; | |
| import javax.mail.URLName; | |
| import javax.mail.internet.InternetAddress; | |
| import javax.naming.ConfigurationException; | |
| import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; | |
| import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; | |
| import org.apache.log4j.Logger; | |
| import org.springframework.stereotype.Component; | |
| import com.cloud.alert.dao.AlertDao; | |
| import com.cloud.api.ApiDBUtils; | |
| import com.cloud.capacity.Capacity; | |
| import com.cloud.capacity.CapacityManager; | |
| import com.cloud.capacity.CapacityState; | |
| import com.cloud.capacity.CapacityVO; | |
| import com.cloud.capacity.dao.CapacityDao; | |
| import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity; | |
| import com.cloud.configuration.Config; | |
| import com.cloud.configuration.ConfigurationManager; | |
| import com.cloud.configuration.dao.ConfigurationDao; | |
| import com.cloud.dc.ClusterVO; | |
| import com.cloud.dc.DataCenter.NetworkType; | |
| import com.cloud.dc.DataCenterVO; | |
| import com.cloud.dc.HostPodVO; | |
| import com.cloud.dc.Vlan.VlanType; | |
| import com.cloud.dc.dao.ClusterDao; | |
| import com.cloud.dc.dao.DataCenterDao; | |
| import com.cloud.dc.dao.DataCenterIpAddressDao; | |
| import com.cloud.dc.dao.HostPodDao; | |
| import com.cloud.event.AlertGenerator; | |
| import com.cloud.host.Host; | |
| import com.cloud.host.HostVO; | |
| import com.cloud.host.dao.HostDao; | |
| import com.cloud.network.dao.IPAddressDao; | |
| import com.cloud.org.Grouping.AllocationState; | |
| import com.cloud.resource.ResourceManager; | |
| import com.cloud.storage.StorageManager; | |
| import com.cloud.storage.dao.VolumeDao; | |
| import com.cloud.utils.NumbersUtil; | |
| import com.cloud.utils.component.ManagerBase; | |
| import com.cloud.utils.db.DB; | |
| import com.cloud.utils.db.SearchCriteria; | |
| import com.sun.mail.smtp.SMTPMessage; | |
| import com.sun.mail.smtp.SMTPSSLTransport; | |
| import com.sun.mail.smtp.SMTPTransport; | |
| @Component | |
| @Local(value={AlertManager.class}) | |
| public class AlertManagerImpl extends ManagerBase implements AlertManager { | |
| private static final Logger s_logger = Logger.getLogger(AlertManagerImpl.class.getName()); | |
| private static final Logger s_alertsLogger = Logger.getLogger("org.apache.cloudstack.alerts"); | |
| private static final long INITIAL_CAPACITY_CHECK_DELAY = 30L * 1000L; // thirty seconds expressed in milliseconds | |
| private static final DecimalFormat _dfPct = new DecimalFormat("###.##"); | |
| private static final DecimalFormat _dfWhole = new DecimalFormat("########"); | |
| private EmailAlert _emailAlert; | |
| @Inject private AlertDao _alertDao; | |
| @Inject private HostDao _hostDao; | |
| @Inject protected StorageManager _storageMgr; | |
| @Inject protected CapacityManager _capacityMgr; | |
| @Inject private CapacityDao _capacityDao; | |
| @Inject private DataCenterDao _dcDao; | |
| @Inject private HostPodDao _podDao; | |
| @Inject private ClusterDao _clusterDao; | |
| @Inject private VolumeDao _volumeDao; | |
| @Inject private IPAddressDao _publicIPAddressDao; | |
| @Inject private DataCenterIpAddressDao _privateIPAddressDao; | |
| @Inject private PrimaryDataStoreDao _storagePoolDao; | |
| @Inject private ConfigurationDao _configDao; | |
| @Inject private ResourceManager _resourceMgr; | |
| @Inject private ConfigurationManager _configMgr; | |
| private Timer _timer = null; | |
| private float _cpuOverProvisioningFactor = 1; | |
| private long _capacityCheckPeriod = 60L * 60L * 1000L; // one hour by default | |
| private double _memoryCapacityThreshold = 0.75; | |
| private double _cpuCapacityThreshold = 0.75; | |
| private double _storageCapacityThreshold = 0.75; | |
| private double _storageAllocCapacityThreshold = 0.75; | |
| private double _publicIPCapacityThreshold = 0.75; | |
| private double _privateIPCapacityThreshold = 0.75; | |
| private double _secondaryStorageCapacityThreshold = 0.75; | |
| private double _vlanCapacityThreshold = 0.75; | |
| private double _directNetworkPublicIpCapacityThreshold = 0.75; | |
| private double _localStorageCapacityThreshold = 0.75; | |
| Map<Short,Double> _capacityTypeThresholdMap = new HashMap<Short, Double>(); | |
| @Override | |
| public boolean configure(String name, Map<String, Object> params) throws ConfigurationException { | |
| Map<String, String> configs = _configDao.getConfiguration("management-server", params); | |
| // set up the email system for alerts | |
| String emailAddressList = configs.get("alert.email.addresses"); | |
| String[] emailAddresses = null; | |
| if (emailAddressList != null) { | |
| emailAddresses = emailAddressList.split(","); | |
| } | |
| String smtpHost = configs.get("alert.smtp.host"); | |
| int smtpPort = NumbersUtil.parseInt(configs.get("alert.smtp.port"), 25); | |
| String useAuthStr = configs.get("alert.smtp.useAuth"); | |
| boolean useAuth = ((useAuthStr == null) ? false : Boolean.parseBoolean(useAuthStr)); | |
| String smtpUsername = configs.get("alert.smtp.username"); | |
| String smtpPassword = configs.get("alert.smtp.password"); | |
| String emailSender = configs.get("alert.email.sender"); | |
| String smtpDebugStr = configs.get("alert.smtp.debug"); | |
| boolean smtpDebug = false; | |
| if (smtpDebugStr != null) { | |
| smtpDebug = Boolean.parseBoolean(smtpDebugStr); | |
| } | |
| _emailAlert = new EmailAlert(emailAddresses, smtpHost, smtpPort, useAuth, smtpUsername, smtpPassword, emailSender, smtpDebug); | |
| String storageCapacityThreshold = _configDao.getValue(Config.StorageCapacityThreshold.key()); | |
| String cpuCapacityThreshold = _configDao.getValue(Config.CPUCapacityThreshold.key()); | |
| String memoryCapacityThreshold = _configDao.getValue(Config.MemoryCapacityThreshold.key()); | |
| String storageAllocCapacityThreshold = _configDao.getValue(Config.StorageAllocatedCapacityThreshold.key()); | |
| String publicIPCapacityThreshold = _configDao.getValue(Config.PublicIpCapacityThreshold.key()); | |
| String privateIPCapacityThreshold = _configDao.getValue(Config.PrivateIpCapacityThreshold.key()); | |
| String secondaryStorageCapacityThreshold = _configDao.getValue(Config.SecondaryStorageCapacityThreshold.key()); | |
| String vlanCapacityThreshold = _configDao.getValue(Config.VlanCapacityThreshold.key()); | |
| String directNetworkPublicIpCapacityThreshold = _configDao.getValue(Config.DirectNetworkPublicIpCapacityThreshold.key()); | |
| String localStorageCapacityThreshold = _configDao.getValue(Config.LocalStorageCapacityThreshold.key()); | |
| if (storageCapacityThreshold != null) { | |
| _storageCapacityThreshold = Double.parseDouble(storageCapacityThreshold); | |
| } | |
| if (storageAllocCapacityThreshold != null) { | |
| _storageAllocCapacityThreshold = Double.parseDouble(storageAllocCapacityThreshold); | |
| } | |
| if (cpuCapacityThreshold != null) { | |
| _cpuCapacityThreshold = Double.parseDouble(cpuCapacityThreshold); | |
| } | |
| if (memoryCapacityThreshold != null) { | |
| _memoryCapacityThreshold = Double.parseDouble(memoryCapacityThreshold); | |
| } | |
| if (publicIPCapacityThreshold != null) { | |
| _publicIPCapacityThreshold = Double.parseDouble(publicIPCapacityThreshold); | |
| } | |
| if (privateIPCapacityThreshold != null) { | |
| _privateIPCapacityThreshold = Double.parseDouble(privateIPCapacityThreshold); | |
| } | |
| if (secondaryStorageCapacityThreshold != null) { | |
| _secondaryStorageCapacityThreshold = Double.parseDouble(secondaryStorageCapacityThreshold); | |
| } | |
| if (vlanCapacityThreshold != null) { | |
| _vlanCapacityThreshold = Double.parseDouble(vlanCapacityThreshold); | |
| } | |
| if (directNetworkPublicIpCapacityThreshold != null) { | |
| _directNetworkPublicIpCapacityThreshold = Double.parseDouble(directNetworkPublicIpCapacityThreshold); | |
| } | |
| if (localStorageCapacityThreshold != null) { | |
| _localStorageCapacityThreshold = Double.parseDouble(localStorageCapacityThreshold); | |
| } | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_STORAGE, _storageCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, _storageAllocCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_CPU, _cpuCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_MEMORY, _memoryCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP, _publicIPCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_PRIVATE_IP, _privateIPCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE, _secondaryStorageCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VLAN, _vlanCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP, _directNetworkPublicIpCapacityThreshold); | |
| _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_LOCAL_STORAGE, _localStorageCapacityThreshold); | |
| String capacityCheckPeriodStr = configs.get("capacity.check.period"); | |
| if (capacityCheckPeriodStr != null) { | |
| _capacityCheckPeriod = Long.parseLong(capacityCheckPeriodStr); | |
| if(_capacityCheckPeriod <= 0) | |
| _capacityCheckPeriod = Long.parseLong(Config.CapacityCheckPeriod.getDefaultValue()); | |
| } | |
| String cpuOverProvisioningFactorStr = configs.get("cpu.overprovisioning.factor"); | |
| if (cpuOverProvisioningFactorStr != null) { | |
| _cpuOverProvisioningFactor = NumbersUtil.parseFloat(cpuOverProvisioningFactorStr,1); | |
| if(_cpuOverProvisioningFactor < 1){ | |
| _cpuOverProvisioningFactor = 1; | |
| } | |
| } | |
| _timer = new Timer("CapacityChecker"); | |
| return true; | |
| } | |
| @Override | |
| public boolean start() { | |
| _timer.schedule(new CapacityChecker(), INITIAL_CAPACITY_CHECK_DELAY, _capacityCheckPeriod); | |
| return true; | |
| } | |
| @Override | |
| public boolean stop() { | |
| _timer.cancel(); | |
| return true; | |
| } | |
| @Override | |
| public void clearAlert(short alertType, long dataCenterId, long podId) { | |
| try { | |
| if (_emailAlert != null) { | |
| _emailAlert.clearAlert(alertType, dataCenterId, podId); | |
| } | |
| } catch (Exception ex) { | |
| s_logger.error("Problem clearing email alert", ex); | |
| } | |
| } | |
| @Override | |
| public void sendAlert(short alertType, long dataCenterId, Long podId, String subject, String body) { | |
| // publish alert | |
| AlertGenerator.publishAlertOnEventBus(getAlertType(alertType), dataCenterId, podId, subject, body); | |
| // TODO: queue up these messages and send them as one set of issues once a certain number of issues is reached? If that's the case, | |
| // shouldn't we have a type/severity as part of the API so that severe errors get sent right away? | |
| try { | |
| if (_emailAlert != null) { | |
| _emailAlert.sendAlert(alertType, dataCenterId, podId, null, subject, body); | |
| } else { | |
| s_alertsLogger.warn(" alertType:: " + alertType + " // dataCenterId:: " + dataCenterId + " // podId:: " | |
| + podId + " // clusterId:: " + null + " // message:: " + subject ); | |
| } | |
| } catch (Exception ex) { | |
| s_logger.error("Problem sending email alert", ex); | |
| } | |
| } | |
| private String getAlertType(short alertType) { | |
| if (alertType == ALERT_TYPE_MEMORY) { | |
| return "ALERT.MEMORY"; | |
| } else if (alertType == ALERT_TYPE_CPU) { | |
| return "ALERT.MEMORY"; | |
| } else if (alertType == ALERT_TYPE_STORAGE) { | |
| return "ALERT.STORAGE"; | |
| } else if (alertType == ALERT_TYPE_STORAGE_ALLOCATED) { | |
| return "ALERT.STORAGE.ALLOCATED"; | |
| } else if (alertType == ALERT_TYPE_VIRTUAL_NETWORK_PUBLIC_IP) { | |
| return "ALERT.NETWORK.PUBLICIP"; | |
| } else if (alertType == ALERT_TYPE_PRIVATE_IP) { | |
| return "ALERT.NETWORK.PRIVATEIP"; | |
| } else if (alertType == ALERT_TYPE_SECONDARY_STORAGE) { | |
| return "ALERT.STORAGE.SECONDARY"; | |
| } else if (alertType == ALERT_TYPE_HOST) { | |
| return "ALERT.COMPUTE.HOST"; | |
| } else if (alertType == ALERT_TYPE_USERVM) { | |
| return "ALERT.USERVM"; | |
| } else if (alertType == ALERT_TYPE_DOMAIN_ROUTER) { | |
| return "ALERT.SERVICE.DOMAINROUTER"; | |
| } else if (alertType == ALERT_TYPE_CONSOLE_PROXY) { | |
| return "ALERT.SERVICE.CONSOLEPROXY"; | |
| } else if (alertType == ALERT_TYPE_ROUTING) { | |
| return "ALERT.NETWORK.ROUTING"; | |
| } else if (alertType == ALERT_TYPE_STORAGE_MISC) { | |
| return "ALERT.STORAGE.MISC"; | |
| } else if (alertType == ALERT_TYPE_USAGE_SERVER) { | |
| return "ALERT.USAGE"; | |
| } else if (alertType == ALERT_TYPE_MANAGMENT_NODE) { | |
| return "ALERT.MANAGEMENT"; | |
| } else if (alertType == ALERT_TYPE_DOMAIN_ROUTER_MIGRATE) { | |
| return "ALERT.NETWORK.DOMAINROUTERMIGRATE"; | |
| } else if (alertType == ALERT_TYPE_CONSOLE_PROXY_MIGRATE) { | |
| return "ALERT.SERVICE.CONSOLEPROXYMIGRATE"; | |
| } else if (alertType == ALERT_TYPE_USERVM_MIGRATE) { | |
| return "ALERT.USERVM.MIGRATE"; | |
| } else if (alertType == ALERT_TYPE_VLAN) { | |
| return "ALERT.NETWORK.VLAN"; | |
| } else if (alertType == ALERT_TYPE_SSVM) { | |
| return "ALERT.SERVICE.SSVM"; | |
| } else if (alertType == ALERT_TYPE_USAGE_SERVER_RESULT) { | |
| return "ALERT.USAGE.RESULT"; | |
| } else if (alertType == ALERT_TYPE_STORAGE_DELETE) { | |
| return "ALERT.STORAGE.DELETE"; | |
| } else if (alertType == ALERT_TYPE_UPDATE_RESOURCE_COUNT) { | |
| return "ALERT.RESOURCE.COUNT"; | |
| } else if (alertType == ALERT_TYPE_USAGE_SANITY_RESULT) { | |
| return "ALERT.USAGE.SANITY"; | |
| } else if (alertType == ALERT_TYPE_DIRECT_ATTACHED_PUBLIC_IP) { | |
| return "ALERT.NETWORK.DIRECTPUBLICIP"; | |
| } else if (alertType == ALERT_TYPE_LOCAL_STORAGE) { | |
| return "ALERT.STORAGE.LOCAL"; | |
| } else if (alertType == ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED) { | |
| return "ALERT.RESOURCE.EXCEED"; | |
| } | |
| return "UNKNOWN"; | |
| } | |
| @Override @DB | |
| public void recalculateCapacity() { | |
| // FIXME: the right way to do this is to register a listener (see RouterStatsListener, VMSyncListener) | |
| // for the vm sync state. The listener model has connects/disconnects to keep things in sync much better | |
| // than this model right now, so when a VM is started, we update the amount allocated, and when a VM | |
| // is stopped we updated the amount allocated, and when VM sync reports a changed state, we update | |
| // the amount allocated. Hopefully it's limited to 3 entry points and will keep the amount allocated | |
| // per host accurate. | |
| try { | |
| if (s_logger.isDebugEnabled()) { | |
| s_logger.debug("recalculating system capacity"); | |
| s_logger.debug("Executing cpu/ram capacity update"); | |
| } | |
| // Calculate CPU and RAM capacities | |
| // get all hosts...even if they are not in 'UP' state | |
| List<HostVO> hosts = _resourceMgr.listAllNotInMaintenanceHostsInOneZone(Host.Type.Routing, null); | |
| for (HostVO host : hosts) { | |
| _capacityMgr.updateCapacityForHost(host); | |
| } | |
| if (s_logger.isDebugEnabled()) { | |
| s_logger.debug("Done executing cpu/ram capacity update"); | |
| s_logger.debug("Executing storage capacity update"); | |
| } | |
| // Calculate storage pool capacity | |
| List<StoragePoolVO> storagePools = _storagePoolDao.listAll(); | |
| for (StoragePoolVO pool : storagePools) { | |
| long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null); | |
| if (pool.isShared()){ | |
| _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk); | |
| }else { | |
| _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk); | |
| } | |
| } | |
| if (s_logger.isDebugEnabled()) { | |
| s_logger.debug("Done executing storage capacity update"); | |
| s_logger.debug("Executing capacity updates for public ip and Vlans"); | |
| } | |
| List<DataCenterVO> datacenters = _dcDao.listAll(); | |
| for (DataCenterVO datacenter : datacenters) { | |
| long dcId = datacenter.getId(); | |
| //NOTE | |
| //What happens if we have multiple vlans? Dashboard currently shows stats | |
| //with no filter based on a vlan | |
| //ideal way would be to remove out the vlan param, and filter only on dcId | |
| //implementing the same | |
| // Calculate new Public IP capacity for Virtual Network | |
| if (datacenter.getNetworkType() == NetworkType.Advanced){ | |
| createOrUpdateIpCapacity(dcId, null, CapacityVO.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP, datacenter.getAllocationState()); | |
| } | |
| // Calculate new Public IP capacity for Direct Attached Network | |
| createOrUpdateIpCapacity(dcId, null, CapacityVO.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP, datacenter.getAllocationState()); | |
| if (datacenter.getNetworkType() == NetworkType.Advanced){ | |
| //Calculate VLAN's capacity | |
| createOrUpdateVlanCapacity(dcId, datacenter.getAllocationState()); | |
| } | |
| } | |
| if (s_logger.isDebugEnabled()) { | |
| s_logger.debug("Done capacity updates for public ip and Vlans"); | |
| s_logger.debug("Executing capacity updates for private ip"); | |
| } | |
| // Calculate new Private IP capacity | |
| List<HostPodVO> pods = _podDao.listAll(); | |
| for (HostPodVO pod : pods) { | |
| long podId = pod.getId(); | |
| long dcId = pod.getDataCenterId(); | |
| createOrUpdateIpCapacity(dcId, podId, CapacityVO.CAPACITY_TYPE_PRIVATE_IP, _configMgr.findPodAllocationState(pod)); | |
| } | |
| if (s_logger.isDebugEnabled()) { | |
| s_logger.debug("Done executing capacity updates for private ip"); | |
| s_logger.debug("Done recalculating system capacity"); | |
| } | |
| } catch (Throwable t) { | |
| s_logger.error("Caught exception in recalculating capacity", t); | |
| } | |
| } | |
| private void createOrUpdateVlanCapacity(long dcId, AllocationState capacityState) { | |
| SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria(); | |
| List<CapacityVO> capacities = _capacityDao.search(capacitySC, null); | |
| capacitySC = _capacityDao.createSearchCriteria(); | |
| capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId); | |
| capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, Capacity.CAPACITY_TYPE_VLAN); | |
| capacities = _capacityDao.search(capacitySC, null); | |
| int totalVlans = _dcDao.countZoneVlans(dcId, false); | |
| int allocatedVlans = _dcDao.countZoneVlans(dcId, true); | |
| if (capacities.size() == 0){ | |
| CapacityVO newVlanCapacity = new CapacityVO(null, dcId, null, null, allocatedVlans, totalVlans, Capacity.CAPACITY_TYPE_VLAN); | |
| if (capacityState == AllocationState.Disabled){ | |
| newVlanCapacity.setCapacityState(CapacityState.Disabled); | |
| } | |
| _capacityDao.persist(newVlanCapacity); | |
| }else if ( !(capacities.get(0).getUsedCapacity() == allocatedVlans | |
| && capacities.get(0).getTotalCapacity() == totalVlans) ){ | |
| CapacityVO capacity = capacities.get(0); | |
| capacity.setUsedCapacity(allocatedVlans); | |
| capacity.setTotalCapacity(totalVlans); | |
| _capacityDao.update(capacity.getId(), capacity); | |
| } | |
| } | |
| public void createOrUpdateIpCapacity(Long dcId, Long podId, short capacityType, AllocationState capacityState){ | |
| SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria(); | |
| List<CapacityVO> capacities = _capacityDao.search(capacitySC, null); | |
| capacitySC = _capacityDao.createSearchCriteria(); | |
| capacitySC.addAnd("podId", SearchCriteria.Op.EQ, podId); | |
| capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId); | |
| capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType); | |
| int totalIPs; | |
| int allocatedIPs; | |
| capacities = _capacityDao.search(capacitySC, null); | |
| if (capacityType == CapacityVO.CAPACITY_TYPE_PRIVATE_IP){ | |
| totalIPs = _privateIPAddressDao.countIPs(podId, dcId, false); | |
| allocatedIPs = _privateIPAddressDao.countIPs(podId, dcId, true); | |
| }else if (capacityType == CapacityVO.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP){ | |
| totalIPs = _publicIPAddressDao.countIPsForNetwork(dcId, false, VlanType.VirtualNetwork); | |
| allocatedIPs = _publicIPAddressDao.countIPsForNetwork(dcId, true, VlanType.VirtualNetwork); | |
| }else { | |
| totalIPs = _publicIPAddressDao.countIPsForNetwork(dcId, false, VlanType.DirectAttached); | |
| allocatedIPs = _publicIPAddressDao.countIPsForNetwork(dcId, true, VlanType.DirectAttached); | |
| } | |
| if (capacities.size() == 0){ | |
| CapacityVO newPublicIPCapacity = new CapacityVO(null, dcId, podId, null, allocatedIPs, totalIPs, capacityType); | |
| if (capacityState == AllocationState.Disabled){ | |
| newPublicIPCapacity.setCapacityState(CapacityState.Disabled); | |
| } | |
| _capacityDao.persist(newPublicIPCapacity); | |
| }else if ( !(capacities.get(0).getUsedCapacity() == allocatedIPs | |
| && capacities.get(0).getTotalCapacity() == totalIPs) ){ | |
| CapacityVO capacity = capacities.get(0); | |
| capacity.setUsedCapacity(allocatedIPs); | |
| capacity.setTotalCapacity(totalIPs); | |
| _capacityDao.update(capacity.getId(), capacity); | |
| } | |
| } | |
| class CapacityChecker extends TimerTask { | |
| @Override | |
| public void run() { | |
| try { | |
| s_logger.debug("Running Capacity Checker ... "); | |
| checkForAlerts(); | |
| s_logger.debug("Done running Capacity Checker ... "); | |
| } catch (Throwable t) { | |
| s_logger.error("Exception in CapacityChecker", t); | |
| } | |
| } | |
| } | |
| public void checkForAlerts(){ | |
| recalculateCapacity(); | |
| // abort if we can't possibly send an alert... | |
| if (_emailAlert == null) { | |
| return; | |
| } | |
| //Get all datacenters, pods and clusters in the system. | |
| List<DataCenterVO> dataCenterList = _dcDao.listAll(); | |
| List<ClusterVO> clusterList = _clusterDao.listAll(); | |
| List<HostPodVO> podList = _podDao.listAll(); | |
| //Get capacity types at different levels | |
| List<Short> dataCenterCapacityTypes = getCapacityTypesAtZoneLevel(); | |
| List<Short> podCapacityTypes = getCapacityTypesAtPodLevel(); | |
| List<Short> clusterCapacityTypes = getCapacityTypesAtClusterLevel(); | |
| // Generate Alerts for Zone Level capacities | |
| for(DataCenterVO dc : dataCenterList){ | |
| for (Short capacityType : dataCenterCapacityTypes){ | |
| List<SummedCapacity> capacity = new ArrayList<SummedCapacity>(); | |
| capacity = _capacityDao.findCapacityBy(capacityType.intValue(), dc.getId(), null, null); | |
| if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE){ | |
| capacity.add(getUsedStats(capacityType, dc.getId(), null, null)); | |
| } | |
| if (capacity == null || capacity.size() == 0){ | |
| continue; | |
| } | |
| double totalCapacity = capacity.get(0).getTotalCapacity(); | |
| double usedCapacity = capacity.get(0).getUsedCapacity(); | |
| if (totalCapacity != 0 && usedCapacity/totalCapacity > _capacityTypeThresholdMap.get(capacityType)){ | |
| generateEmailAlert(dc, null, null, totalCapacity, usedCapacity, capacityType); | |
| } | |
| } | |
| } | |
| // Generate Alerts for Pod Level capacities | |
| for( HostPodVO pod : podList){ | |
| for (Short capacityType : podCapacityTypes){ | |
| List<SummedCapacity> capacity = _capacityDao.findCapacityBy(capacityType.intValue(), pod.getDataCenterId(), pod.getId(), null); | |
| if (capacity == null || capacity.size() == 0){ | |
| continue; | |
| } | |
| double totalCapacity = capacity.get(0).getTotalCapacity(); | |
| double usedCapacity = capacity.get(0).getUsedCapacity(); | |
| if (totalCapacity != 0 && usedCapacity/totalCapacity > _capacityTypeThresholdMap.get(capacityType)){ | |
| generateEmailAlert(ApiDBUtils.findZoneById(pod.getDataCenterId()), pod, null, | |
| totalCapacity, usedCapacity, capacityType); | |
| } | |
| } | |
| } | |
| // Generate Alerts for Cluster Level capacities | |
| for( ClusterVO cluster : clusterList){ | |
| for (Short capacityType : clusterCapacityTypes){ | |
| List<SummedCapacity> capacity = new ArrayList<SummedCapacity>(); | |
| float overProvFactor = 1f; | |
| capacity = _capacityDao.findCapacityBy(capacityType.intValue(), cluster.getDataCenterId(), null, cluster.getId()); | |
| if (capacityType == Capacity.CAPACITY_TYPE_STORAGE){ | |
| capacity.add(getUsedStats(capacityType, cluster.getDataCenterId(), cluster.getPodId(), cluster.getId())); | |
| } | |
| if (capacity == null || capacity.size() == 0){ | |
| continue; | |
| } | |
| if (capacityType == Capacity.CAPACITY_TYPE_CPU){ | |
| overProvFactor = ApiDBUtils.getCpuOverprovisioningFactor(); | |
| } | |
| double totalCapacity = capacity.get(0).getTotalCapacity() * overProvFactor; | |
| double usedCapacity = capacity.get(0).getUsedCapacity() + capacity.get(0).getReservedCapacity(); | |
| if (totalCapacity != 0 && usedCapacity/totalCapacity > _capacityTypeThresholdMap.get(capacityType)){ | |
| generateEmailAlert(ApiDBUtils.findZoneById(cluster.getDataCenterId()), ApiDBUtils.findPodById(cluster.getPodId()), cluster, | |
| totalCapacity, usedCapacity, capacityType); | |
| } | |
| } | |
| } | |
| } | |
| private SummedCapacity getUsedStats(short capacityType, long zoneId, Long podId, Long clusterId){ | |
| CapacityVO capacity; | |
| if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE){ | |
| capacity = _storageMgr.getSecondaryStorageUsedStats(null, zoneId); | |
| }else{ | |
| capacity = _storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId); | |
| } | |
| if (capacity != null){ | |
| return new SummedCapacity(capacity.getUsedCapacity(), 0, capacity.getTotalCapacity(), capacityType, clusterId, podId); | |
| }else{ | |
| return null; | |
| } | |
| } | |
| private void generateEmailAlert(DataCenterVO dc, HostPodVO pod, ClusterVO cluster, double totalCapacity, double usedCapacity, short capacityType){ | |
| String msgSubject = null; | |
| String msgContent = null; | |
| String totalStr; | |
| String usedStr; | |
| String pctStr = formatPercent(usedCapacity/totalCapacity); | |
| short alertType = -1; | |
| Long podId = pod == null ? null : pod.getId(); | |
| Long clusterId = cluster == null ? null : cluster.getId(); | |
| switch (capacityType) { | |
| //Cluster Level | |
| case CapacityVO.CAPACITY_TYPE_MEMORY: | |
| msgSubject = "System Alert: Low Available Memory in cluster " +cluster.getName()+ " pod " +pod.getName()+ " of availability zone " + dc.getName(); | |
| totalStr = formatBytesToMegabytes(totalCapacity); | |
| usedStr = formatBytesToMegabytes(usedCapacity); | |
| msgContent = "System memory is low, total: " + totalStr + " MB, used: " + usedStr + " MB (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_MEMORY; | |
| break; | |
| case CapacityVO.CAPACITY_TYPE_CPU: | |
| msgSubject = "System Alert: Low Unallocated CPU in cluster " +cluster.getName()+ " pod " +pod.getName()+ " of availability zone " + dc.getName(); | |
| totalStr = _dfWhole.format(totalCapacity); | |
| usedStr = _dfWhole.format(usedCapacity); | |
| msgContent = "Unallocated CPU is low, total: " + totalStr + " Mhz, used: " + usedStr + " Mhz (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_CPU; | |
| break; | |
| case CapacityVO.CAPACITY_TYPE_STORAGE: | |
| msgSubject = "System Alert: Low Available Storage in cluster " +cluster.getName()+ " pod " +pod.getName()+ " of availability zone " + dc.getName(); | |
| totalStr = formatBytesToMegabytes(totalCapacity); | |
| usedStr = formatBytesToMegabytes(usedCapacity); | |
| msgContent = "Available storage space is low, total: " + totalStr + " MB, used: " + usedStr + " MB (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_STORAGE; | |
| break; | |
| case CapacityVO.CAPACITY_TYPE_STORAGE_ALLOCATED: | |
| msgSubject = "System Alert: Remaining unallocated Storage is low in cluster " +cluster.getName()+ " pod " +pod.getName()+ " of availability zone " + dc.getName(); | |
| totalStr = formatBytesToMegabytes(totalCapacity); | |
| usedStr = formatBytesToMegabytes(usedCapacity); | |
| msgContent = "Unallocated storage space is low, total: " + totalStr + " MB, allocated: " + usedStr + " MB (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_STORAGE_ALLOCATED; | |
| break; | |
| case CapacityVO.CAPACITY_TYPE_LOCAL_STORAGE: | |
| msgSubject = "System Alert: Remaining unallocated Local Storage is low in cluster " +cluster.getName()+ " pod " +pod.getName()+ " of availability zone " + dc.getName(); | |
| totalStr = formatBytesToMegabytes(totalCapacity); | |
| usedStr = formatBytesToMegabytes(usedCapacity); | |
| msgContent = "Unallocated storage space is low, total: " + totalStr + " MB, allocated: " + usedStr + " MB (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_LOCAL_STORAGE; | |
| break; | |
| //Pod Level | |
| case CapacityVO.CAPACITY_TYPE_PRIVATE_IP: | |
| msgSubject = "System Alert: Number of unallocated private IPs is low in pod " +pod.getName()+ " of availability zone " + dc.getName(); | |
| totalStr = Double.toString(totalCapacity); | |
| usedStr = Double.toString(usedCapacity); | |
| msgContent = "Number of unallocated private IPs is low, total: " + totalStr + ", allocated: " + usedStr + " (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_PRIVATE_IP; | |
| break; | |
| //Zone Level | |
| case CapacityVO.CAPACITY_TYPE_SECONDARY_STORAGE: | |
| msgSubject = "System Alert: Low Available Secondary Storage in availability zone " + dc.getName(); | |
| totalStr = formatBytesToMegabytes(totalCapacity); | |
| usedStr = formatBytesToMegabytes(usedCapacity); | |
| msgContent = "Available secondary storage space is low, total: " + totalStr + " MB, used: " + usedStr + " MB (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_SECONDARY_STORAGE; | |
| break; | |
| case CapacityVO.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP: | |
| msgSubject = "System Alert: Number of unallocated virtual network public IPs is low in availability zone " + dc.getName(); | |
| totalStr = Double.toString(totalCapacity); | |
| usedStr = Double.toString(usedCapacity); | |
| msgContent = "Number of unallocated public IPs is low, total: " + totalStr + ", allocated: " + usedStr + " (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_VIRTUAL_NETWORK_PUBLIC_IP; | |
| break; | |
| case CapacityVO.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP: | |
| msgSubject = "System Alert: Number of unallocated shared network IPs is low in availability zone " + dc.getName(); | |
| totalStr = Double.toString(totalCapacity); | |
| usedStr = Double.toString(usedCapacity); | |
| msgContent = "Number of unallocated shared network IPs is low, total: " + totalStr + ", allocated: " + usedStr + " (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_DIRECT_ATTACHED_PUBLIC_IP; | |
| break; | |
| case CapacityVO.CAPACITY_TYPE_VLAN: | |
| msgSubject = "System Alert: Number of unallocated VLANs is low in availability zone " + dc.getName(); | |
| totalStr = Double.toString(totalCapacity); | |
| usedStr = Double.toString(usedCapacity); | |
| msgContent = "Number of unallocated VLANs is low, total: " + totalStr + ", allocated: " + usedStr + " (" + pctStr + "%)"; | |
| alertType = ALERT_TYPE_VLAN; | |
| break; | |
| } | |
| try { | |
| if (s_logger.isDebugEnabled()){ | |
| s_logger.debug(msgSubject); | |
| s_logger.debug(msgContent); | |
| } | |
| _emailAlert.sendAlert(alertType, dc.getId(), podId, clusterId, msgSubject, msgContent); | |
| } catch (Exception ex) { | |
| s_logger.error("Exception in CapacityChecker", ex); | |
| } | |
| } | |
| private List<Short> getCapacityTypesAtZoneLevel(){ | |
| List<Short> dataCenterCapacityTypes = new ArrayList<Short>(); | |
| dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP); | |
| dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP); | |
| dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE); | |
| dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VLAN); | |
| return dataCenterCapacityTypes; | |
| } | |
| private List<Short> getCapacityTypesAtPodLevel(){ | |
| List<Short> podCapacityTypes = new ArrayList<Short>(); | |
| podCapacityTypes.add(Capacity.CAPACITY_TYPE_PRIVATE_IP); | |
| return podCapacityTypes; | |
| } | |
| private List<Short> getCapacityTypesAtClusterLevel(){ | |
| List<Short> clusterCapacityTypes = new ArrayList<Short>(); | |
| clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_CPU); | |
| clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_MEMORY); | |
| clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_STORAGE); | |
| clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED); | |
| clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_LOCAL_STORAGE); | |
| return clusterCapacityTypes; | |
| } | |
| class EmailAlert { | |
| private Session _smtpSession; | |
| private InternetAddress[] _recipientList; | |
| private final String _smtpHost; | |
| private int _smtpPort = -1; | |
| private boolean _smtpUseAuth = false; | |
| private final String _smtpUsername; | |
| private final String _smtpPassword; | |
| private final String _emailSender; | |
| public EmailAlert(String[] recipientList, String smtpHost, int smtpPort, boolean smtpUseAuth, final String smtpUsername, final String smtpPassword, String emailSender, boolean smtpDebug) { | |
| if (recipientList != null) { | |
| _recipientList = new InternetAddress[recipientList.length]; | |
| for (int i = 0; i < recipientList.length; i++) { | |
| try { | |
| _recipientList[i] = new InternetAddress(recipientList[i], recipientList[i]); | |
| } catch (Exception ex) { | |
| s_logger.error("Exception creating address for: " + recipientList[i], ex); | |
| } | |
| } | |
| } | |
| _smtpHost = smtpHost; | |
| _smtpPort = smtpPort; | |
| _smtpUseAuth = smtpUseAuth; | |
| _smtpUsername = smtpUsername; | |
| _smtpPassword = smtpPassword; | |
| _emailSender = emailSender; | |
| if (_smtpHost != null) { | |
| Properties smtpProps = new Properties(); | |
| smtpProps.put("mail.smtp.host", smtpHost); | |
| smtpProps.put("mail.smtp.port", smtpPort); | |
| smtpProps.put("mail.smtp.auth", ""+smtpUseAuth); | |
| if (smtpUsername != null) { | |
| smtpProps.put("mail.smtp.user", smtpUsername); | |
| } | |
| smtpProps.put("mail.smtps.host", smtpHost); | |
| smtpProps.put("mail.smtps.port", smtpPort); | |
| smtpProps.put("mail.smtps.auth", ""+smtpUseAuth); | |
| if (smtpUsername != null) { | |
| smtpProps.put("mail.smtps.user", smtpUsername); | |
| } | |
| if ((smtpUsername != null) && (smtpPassword != null)) { | |
| _smtpSession = Session.getInstance(smtpProps, new Authenticator() { | |
| @Override | |
| protected PasswordAuthentication getPasswordAuthentication() { | |
| return new PasswordAuthentication(smtpUsername, smtpPassword); | |
| } | |
| }); | |
| } else { | |
| _smtpSession = Session.getInstance(smtpProps); | |
| } | |
| _smtpSession.setDebug(smtpDebug); | |
| } else { | |
| _smtpSession = null; | |
| } | |
| } | |
| // TODO: make sure this handles SSL transport (useAuth is true) and regular | |
| public void sendAlert(short alertType, long dataCenterId, Long podId, Long clusterId, String subject, String content) throws MessagingException, UnsupportedEncodingException { | |
| s_alertsLogger.warn(" alertType:: " + alertType + " // dataCenterId:: " + dataCenterId + " // podId:: " + | |
| podId + " // clusterId:: " + null + " // message:: " + subject); | |
| AlertVO alert = null; | |
| if ((alertType != AlertManager.ALERT_TYPE_HOST) && | |
| (alertType != AlertManager.ALERT_TYPE_USERVM) && | |
| (alertType != AlertManager.ALERT_TYPE_DOMAIN_ROUTER) && | |
| (alertType != AlertManager.ALERT_TYPE_CONSOLE_PROXY) && | |
| (alertType != AlertManager.ALERT_TYPE_SSVM) && | |
| (alertType != AlertManager.ALERT_TYPE_STORAGE_MISC) && | |
| (alertType != AlertManager.ALERT_TYPE_MANAGMENT_NODE) && | |
| (alertType != AlertManager.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED)) { | |
| alert = _alertDao.getLastAlert(alertType, dataCenterId, podId, clusterId); | |
| } | |
| if (alert == null) { | |
| // set up a new alert | |
| AlertVO newAlert = new AlertVO(); | |
| newAlert.setType(alertType); | |
| newAlert.setSubject(subject); | |
| newAlert.setClusterId(clusterId); | |
| newAlert.setPodId(podId); | |
| newAlert.setDataCenterId(dataCenterId); | |
| newAlert.setSentCount(1); // initialize sent count to 1 since we are now sending an alert | |
| newAlert.setLastSent(new Date()); | |
| _alertDao.persist(newAlert); | |
| } else { | |
| if (s_logger.isDebugEnabled()) { | |
| s_logger.debug("Have already sent: " + alert.getSentCount() + " emails for alert type '" + alertType + "' -- skipping send email"); | |
| } | |
| return; | |
| } | |
| if (_smtpSession != null) { | |
| SMTPMessage msg = new SMTPMessage(_smtpSession); | |
| msg.setSender(new InternetAddress(_emailSender, _emailSender)); | |
| msg.setFrom(new InternetAddress(_emailSender, _emailSender)); | |
| for (InternetAddress address : _recipientList) { | |
| msg.addRecipient(RecipientType.TO, address); | |
| } | |
| msg.setSubject(subject); | |
| msg.setSentDate(new Date()); | |
| msg.setContent(content, "text/plain"); | |
| msg.saveChanges(); | |
| SMTPTransport smtpTrans = null; | |
| if (_smtpUseAuth) { | |
| smtpTrans = new SMTPSSLTransport(_smtpSession, new URLName("smtp", _smtpHost, _smtpPort, null, _smtpUsername, _smtpPassword)); | |
| } else { | |
| smtpTrans = new SMTPTransport(_smtpSession, new URLName("smtp", _smtpHost, _smtpPort, null, _smtpUsername, _smtpPassword)); | |
| } | |
| smtpTrans.connect(); | |
| smtpTrans.sendMessage(msg, msg.getAllRecipients()); | |
| smtpTrans.close(); | |
| } | |
| } | |
| public void clearAlert(short alertType, long dataCenterId, Long podId) { | |
| if (alertType != -1) { | |
| AlertVO alert = _alertDao.getLastAlert(alertType, dataCenterId, podId, null); | |
| if (alert != null) { | |
| AlertVO updatedAlert = _alertDao.createForUpdate(); | |
| updatedAlert.setResolved(new Date()); | |
| _alertDao.update(alert.getId(), updatedAlert); | |
| } | |
| } | |
| } | |
| } | |
| private static String formatPercent(double percentage) { | |
| return _dfPct.format(percentage*100); | |
| } | |
| private static String formatBytesToMegabytes(double bytes) { | |
| double megaBytes = (bytes / (1024 * 1024)); | |
| return _dfWhole.format(megaBytes); | |
| } | |
| } |