server/src/main/java/com/cloud/alert/AlertManagerImpl.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.alert;

import java.io.UnsupportedEncodingException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import javax.inject.Inject;
import javax.mail.MessagingException;
import javax.naming.ConfigurationException;

import com.cloud.dc.DataCenter;
import com.cloud.dc.Pod;
import com.cloud.org.Cluster;

import org.apache.cloudstack.backup.BackupManager;
import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.mailing.MailAddress;
import org.apache.cloudstack.utils.mailing.SMTPMailProperties;
import org.apache.cloudstack.utils.mailing.SMTPMailSender;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;

import com.cloud.alert.dao.AlertDao;
import com.cloud.api.ApiDBUtils;
import com.cloud.capacity.Capacity;
import com.cloud.capacity.CapacityManager;
import com.cloud.capacity.CapacityState;
import com.cloud.capacity.CapacityVO;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenter.NetworkType;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.dc.Vlan.VlanType;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.DataCenterIpAddressDao;
import com.cloud.dc.dao.HostPodDao;
import com.cloud.event.ActionEvent;
import com.cloud.event.AlertGenerator;
import com.cloud.event.EventTypes;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.network.Ipv6Service;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.org.Grouping.AllocationState;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.StorageManager;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;

public class AlertManagerImpl extends ManagerBase implements AlertManager, Configurable {
    protected Logger logger = LogManager.getLogger(AlertManagerImpl.class.getName());

    public static final List<AlertType> ALERTS = Arrays.asList(AlertType.ALERT_TYPE_HOST
            , AlertType.ALERT_TYPE_USERVM
            , AlertType.ALERT_TYPE_DOMAIN_ROUTER
            , AlertType.ALERT_TYPE_CONSOLE_PROXY
            , AlertType.ALERT_TYPE_SSVM
            , AlertType.ALERT_TYPE_STORAGE_MISC
            , AlertType.ALERT_TYPE_MANAGEMENT_NODE
            , AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED
            , AlertType.ALERT_TYPE_UPLOAD_FAILED
            , AlertType.ALERT_TYPE_OOBM_AUTH_ERROR
            , AlertType.ALERT_TYPE_HA_ACTION
            , AlertType.ALERT_TYPE_CA_CERT
            , AlertType.ALERT_TYPE_EXTENSION_PATH_NOT_READY);

    private static final long INITIAL_CAPACITY_CHECK_DELAY = 30L * 1000L; // Thirty seconds expressed in milliseconds.

    private static final DecimalFormat DfPct = new DecimalFormat("###.##");
    private static final DecimalFormat DfWhole = new DecimalFormat("########");

    @Inject
    AlertDao _alertDao;
    @Inject
    protected StorageManager _storageMgr;
    @Inject
    protected CapacityManager _capacityMgr;
    @Inject
    CapacityDao _capacityDao;
    @Inject
    DataCenterDao _dcDao;
    @Inject
    HostPodDao _podDao;
    @Inject
    ClusterDao _clusterDao;
    @Inject
    IPAddressDao _publicIPAddressDao;
    @Inject
    DataCenterIpAddressDao _privateIPAddressDao;
    @Inject
    PrimaryDataStoreDao _storagePoolDao;
    @Inject
    ConfigurationDao _configDao;
    @Inject
    ResourceManager _resourceMgr;
    @Inject
    ConfigurationManager _configMgr;
    @Inject
    protected BackupManager backupManager;
    @Inject
    protected ConfigDepot _configDepot;
    @Inject
    private ObjectStoreDao _objectStoreDao;
    @Inject
    Ipv6Service ipv6Service;
    @Inject
    HostDao hostDao;

    private Timer _timer = null;
    private long _capacityCheckPeriod = 60L * 60L * 1000L; // One hour by default.
    private double _publicIPCapacityThreshold = 0.75;
    private double _privateIPCapacityThreshold = 0.75;
    private double _secondaryStorageCapacityThreshold = 0.75;
    private double _vlanCapacityThreshold = 0.75;
    private double _directNetworkPublicIpCapacityThreshold = 0.75;
    private double _localStorageCapacityThreshold = 0.75;
    private double _backupStorageCapacityThreshold = 0.75;
    private double _objectStorageCapacityThreshold = 0.75;
    Map<Short, Double> _capacityTypeThresholdMap = new HashMap<>();

    private final ExecutorService _executor;

    protected SMTPMailSender mailSender;
    protected String[] recipients = null;
    protected String senderAddress = null;

    public AlertManagerImpl() {
        _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Email-Alerts-Sender"));
    }

    @Override
    public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
        Map<String, String> configs = _configDao.getConfiguration("management-server", params);

        // set up the email system for alerts
        String emailAddressList = configs.get("alert.email.addresses");
        if (emailAddressList != null) {
            recipients = emailAddressList.split(",");
        }

        senderAddress = configs.get("alert.email.sender");

        String namespace = "alert.smtp";
        String timeoutConfig = String.format("%s.timeout", namespace);
        String connectionTimeoutConfig = String.format("%s.connectiontimeout", namespace);

        int smtpTimeout = NumberUtils.toInt(configs.get(timeoutConfig), 30000);
        int smtpConnectionTimeout = NumberUtils.toInt(configs.get(connectionTimeoutConfig), 30000);

        configs.put(timeoutConfig, String.valueOf(smtpTimeout));
        configs.put(connectionTimeoutConfig, String.valueOf(smtpConnectionTimeout));

        mailSender = new SMTPMailSender(configs, namespace);

        String publicIPCapacityThreshold = _configDao.getValue(Config.PublicIpCapacityThreshold.key());
        String privateIPCapacityThreshold = _configDao.getValue(Config.PrivateIpCapacityThreshold.key());
        String secondaryStorageCapacityThreshold = _configDao.getValue(Config.SecondaryStorageCapacityThreshold.key());
        String vlanCapacityThreshold = _configDao.getValue(Config.VlanCapacityThreshold.key());
        String directNetworkPublicIpCapacityThreshold = _configDao.getValue(Config.DirectNetworkPublicIpCapacityThreshold.key());
        String localStorageCapacityThreshold = _configDao.getValue(Config.LocalStorageCapacityThreshold.key());
        String backupStorageCapacityThreshold = _configDao.getValue(BackupManager.BackupStorageCapacityThreshold.key());
        String objectStorageCapacityThreshold = _configDao.getValue(_storageMgr.ObjectStorageCapacityThreshold.key());

        if (publicIPCapacityThreshold != null) {
            _publicIPCapacityThreshold = Double.parseDouble(publicIPCapacityThreshold);
        }
        if (privateIPCapacityThreshold != null) {
            _privateIPCapacityThreshold = Double.parseDouble(privateIPCapacityThreshold);
        }
        if (secondaryStorageCapacityThreshold != null) {
            _secondaryStorageCapacityThreshold = Double.parseDouble(secondaryStorageCapacityThreshold);
        }
        if (vlanCapacityThreshold != null) {
            _vlanCapacityThreshold = Double.parseDouble(vlanCapacityThreshold);
        }
        if (directNetworkPublicIpCapacityThreshold != null) {
            _directNetworkPublicIpCapacityThreshold = Double.parseDouble(directNetworkPublicIpCapacityThreshold);
        }
        if (localStorageCapacityThreshold != null) {
            _localStorageCapacityThreshold = Double.parseDouble(localStorageCapacityThreshold);
        }
        if (backupStorageCapacityThreshold != null) {
            _backupStorageCapacityThreshold = Double.parseDouble(backupStorageCapacityThreshold);
        }
        if (objectStorageCapacityThreshold != null) {
            _objectStorageCapacityThreshold = Double.parseDouble(objectStorageCapacityThreshold);
        }

        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP, _publicIPCapacityThreshold);
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_PRIVATE_IP, _privateIPCapacityThreshold);
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE, _secondaryStorageCapacityThreshold);
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VLAN, _vlanCapacityThreshold);
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP, _directNetworkPublicIpCapacityThreshold);
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_LOCAL_STORAGE, _localStorageCapacityThreshold);
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET, Ipv6SubnetCapacityThreshold.value());
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, _backupStorageCapacityThreshold);
        _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, _objectStorageCapacityThreshold);

        String capacityCheckPeriodStr = configs.get("capacity.check.period");
        if (capacityCheckPeriodStr != null) {
            _capacityCheckPeriod = Long.parseLong(capacityCheckPeriodStr);
            if (_capacityCheckPeriod <= 0) {
                _capacityCheckPeriod = Long.parseLong(Config.CapacityCheckPeriod.getDefaultValue());
            }
        }

        _timer = new Timer("CapacityChecker");

        return true;
    }

    @Override
    public boolean start() {
        _timer.schedule(new CapacityChecker(), INITIAL_CAPACITY_CHECK_DELAY, _capacityCheckPeriod);
        return true;
    }

    @Override
    public boolean stop() {
        _timer.cancel();
        return true;
    }

    @Override
    public void clearAlert(AlertType alertType, long dataCenterId, long podId) {
        try {
            clearAlert(alertType.getType(), dataCenterId, podId);
        } catch (Exception ex) {
            logger.error("Problem clearing email alert", ex);
        }
    }

    @Override
    public void sendAlert(AlertType alertType, long dataCenterId, Long podId, String subject, String body) {

        // publish alert
        AlertGenerator.publishAlertOnEventBus(alertType.getName(), dataCenterId, podId, subject, body);

        // TODO:  queue up these messages and send them as one set of issues once a certain number of issues is reached?  If that's the case,
        //         shouldn't we have a type/severity as part of the API so that severe errors get sent right away?
        try {
            if (mailSender != null) {
                sendAlert(alertType, dataCenterId, podId, null, subject, body);
            } else {
                logger.warn("AlertType:: " + alertType + " | dataCenterId:: " + dataCenterId + " | podId:: " + podId +
                        " | message:: " + subject + " | body:: " + body);
            }
        } catch (Exception ex) {
            logger.error("Problem sending email alert", ex);
        }
    }

    /**
     * Recalculates the capacities of hosts, including CPU and RAM.
     */
    protected void recalculateHostCapacities() {
        List<Long> hostIds = hostDao.listIdsByType(Host.Type.Routing);
        if (hostIds.isEmpty()) {
            return;
        }
        ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
        ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
                Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size())));
        for (Long hostId : hostIds) {
            futures.put(hostId, executorService.submit(() -> {
                final HostVO host = hostDao.findById(hostId);
                _capacityMgr.updateCapacityForHost(host);
                return null;
            }));
        }
        for (Map.Entry<Long, Future<Void>> entry: futures.entrySet()) {
            try {
                entry.getValue().get();
            } catch (InterruptedException | ExecutionException e) {
                logger.error(String.format("Error during capacity calculation for host: %d due to : %s",
                        entry.getKey(), e.getMessage()), e);
            }
        }
        executorService.shutdown();
    }

    protected void recalculateStorageCapacities() {
        List<Long> storagePoolIds = _storagePoolDao.listAllIds();
        if (storagePoolIds.isEmpty()) {
            return;
        }
        ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
        ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
                Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size())));
        for (Long poolId: storagePoolIds) {
            futures.put(poolId, executorService.submit(() -> {
                Transaction.execute(new TransactionCallbackNoReturn() {
                    @Override
                    public void doInTransactionWithoutResult(TransactionStatus status) {
                        final StoragePoolVO pool = _storagePoolDao.findById(poolId);
                        long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null);
                        if (pool.isShared()) {
                            _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk);
                        } else {
                            _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk);
                        }
                    }
                });
                return null;
            }));
        }
        for (Map.Entry<Long, Future<Void>> entry: futures.entrySet()) {
            try {
                entry.getValue().get();
            } catch (InterruptedException | ExecutionException e) {
                logger.error(String.format("Error during capacity calculation for storage pool: %d due to : %s",
                        entry.getKey(), e.getMessage()), e);
            }
        }
        executorService.shutdown();
    }

    @Override
    public void recalculateCapacity() {
        // FIXME: the right way to do this is to register a listener (see RouterStatsListener, VMSyncListener)
        //        for the vm sync state.  The listener model has connects/disconnects to keep things in sync much better
        //        than this model right now, so when a VM is started, we update the amount allocated, and when a VM
        //        is stopped we updated the amount allocated, and when VM sync reports a changed state, we update
        //        the amount allocated.  Hopefully it's limited to 3 entry points and will keep the amount allocated
        //        per host accurate.

        try {

            if (logger.isDebugEnabled()) {
                logger.debug("recalculating system capacity");
                logger.debug("Executing cpu/ram capacity update");
            }
            // Calculate CPU and RAM capacities
            recalculateHostCapacities();
            if (logger.isDebugEnabled()) {
                logger.debug("Done executing cpu/ram capacity update");
                logger.debug("Executing storage capacity update");
            }
            // Calculate storage pool capacity
            recalculateStorageCapacities();
            if (logger.isDebugEnabled()) {
                logger.debug("Done executing storage capacity update");
                logger.debug("Executing capacity updates for public ip and Vlans");
            }

            List<DataCenterVO> datacenters = _dcDao.listAll();
            for (DataCenterVO datacenter : datacenters) {
                long dcId = datacenter.getId();

                //NOTE
                //What happens if we have multiple vlans? Dashboard currently shows stats
                //with no filter based on a vlan
                //ideal way would be to remove out the vlan param, and filter only on dcId
                //implementing the same

                // Calculate new Public IP capacity for Virtual Network
                if (datacenter.getNetworkType() == NetworkType.Advanced) {
                    createOrUpdateIpCapacity(dcId, null, Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP, datacenter.getAllocationState());
                    createOrUpdateIpv6Capacity(dcId, datacenter.getAllocationState());
                }

                // Calculate new Public IP capacity for Direct Attached Network
                createOrUpdateIpCapacity(dcId, null, Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP, datacenter.getAllocationState());

                if (datacenter.getNetworkType() == NetworkType.Advanced) {
                    //Calculate VLAN's capacity
                    createOrUpdateVlanCapacity(dcId, datacenter.getAllocationState());
                }
            }

            if (logger.isDebugEnabled()) {
                logger.debug("Done capacity updates for public ip and Vlans");
                logger.debug("Executing capacity updates for private ip");
            }

            // Calculate new Private IP capacity
            List<HostPodVO> pods = _podDao.listAll();
            for (HostPodVO pod : pods) {
                long podId = pod.getId();
                long dcId = pod.getDataCenterId();

                createOrUpdateIpCapacity(dcId, podId, Capacity.CAPACITY_TYPE_PRIVATE_IP, _configMgr.findPodAllocationState(pod));
            }

            if (logger.isDebugEnabled()) {
                logger.debug("Done executing capacity updates for private ip");
                logger.debug("Done recalculating system capacity");
            }

        } catch (Throwable t) {
            logger.error("Caught exception in recalculating capacity", t);
        }
    }

    private void createOrUpdateVlanCapacity(long dcId, AllocationState capacityState) {

        SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
        capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
        capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, Capacity.CAPACITY_TYPE_VLAN);
        List<CapacityVO> capacities = _capacityDao.search(capacitySC, null);

        int totalVlans = _dcDao.countZoneVlans(dcId, false);
        int allocatedVlans = _dcDao.countZoneVlans(dcId, true);

        CapacityState vlanCapacityState = (capacityState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
        if (capacities.isEmpty()) {
            CapacityVO newVlanCapacity = new CapacityVO(null, dcId, null, null, allocatedVlans, totalVlans, Capacity.CAPACITY_TYPE_VLAN);
            newVlanCapacity.setCapacityState(vlanCapacityState);
            _capacityDao.persist(newVlanCapacity);
        } else if (!(capacities.get(0).getUsedCapacity() == allocatedVlans && capacities.get(0).getTotalCapacity() == totalVlans
                && capacities.get(0).getCapacityState() == vlanCapacityState)) {
            CapacityVO capacity = capacities.get(0);
            capacity.setUsedCapacity(allocatedVlans);
            capacity.setTotalCapacity(totalVlans);
            capacity.setCapacityState(vlanCapacityState);
            _capacityDao.update(capacity.getId(), capacity);
        }

    }

    public void createOrUpdateIpCapacity(Long dcId, Long podId, short capacityType, AllocationState capacityState) {
        SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
        capacitySC.addAnd("podId", SearchCriteria.Op.EQ, podId);
        capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
        capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType);

        int totalIPs;
        int allocatedIPs;
        List<CapacityVO> capacities = _capacityDao.search(capacitySC, null);
        if (capacityType == Capacity.CAPACITY_TYPE_PRIVATE_IP) {
            totalIPs = _privateIPAddressDao.countIPs(podId, dcId, false);
            allocatedIPs = _privateIPAddressDao.countIPs(podId, dcId, true);
        } else if (capacityType == Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP) {
            totalIPs = _publicIPAddressDao.countIPsForNetwork(dcId, false, VlanType.VirtualNetwork);
            allocatedIPs = _publicIPAddressDao.countIPsForNetwork(dcId, true, VlanType.VirtualNetwork);
        } else {
            totalIPs = _publicIPAddressDao.countIPsForNetwork(dcId, false, VlanType.DirectAttached);
            allocatedIPs = _publicIPAddressDao.countIPsForNetwork(dcId, true, VlanType.DirectAttached);
        }

        CapacityState ipCapacityState = (capacityState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
        if (capacities.isEmpty()) {
            CapacityVO newPublicIPCapacity = new CapacityVO(null, dcId, podId, null, allocatedIPs, totalIPs, capacityType);
            newPublicIPCapacity.setCapacityState(ipCapacityState);
            _capacityDao.persist(newPublicIPCapacity);
        } else if (!(capacities.get(0).getUsedCapacity() == allocatedIPs && capacities.get(0).getTotalCapacity() == totalIPs
                && capacities.get(0).getCapacityState() == ipCapacityState)) {
            CapacityVO capacity = capacities.get(0);
            capacity.setUsedCapacity(allocatedIPs);
            capacity.setTotalCapacity(totalIPs);
            capacity.setCapacityState(ipCapacityState);
            _capacityDao.update(capacity.getId(), capacity);
        }
    }

    public void createOrUpdateIpv6Capacity(Long dcId, AllocationState capacityState) {
        final short capacityType = Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET;
        SearchCriteria<CapacityVO> capacitySC = _capacityDao.createSearchCriteria();
        capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
        capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType);

        List<CapacityVO>  capacities = _capacityDao.search(capacitySC, null);
        Pair<Integer, Integer> usedTotal =  ipv6Service.getUsedTotalIpv6SubnetForZone(dcId);
        int total = usedTotal.second();
        int allocated = usedTotal.first();
        CapacityState state = (capacityState == AllocationState.Disabled) ? CapacityState.Disabled : CapacityState.Enabled;
        if (capacities.isEmpty()) {
            CapacityVO capacityVO = new CapacityVO(null, dcId, null, null, allocated, total, capacityType);
            capacityVO.setCapacityState(state);
            _capacityDao.persist(capacityVO);
        } else if (!(capacities.get(0).getUsedCapacity() == allocated && capacities.get(0).getTotalCapacity() == total
                && capacities.get(0).getCapacityState() == state)) {
            CapacityVO capacity = capacities.get(0);
            capacity.setUsedCapacity(allocated);
            capacity.setTotalCapacity(total);
            capacity.setCapacityState(state);
            _capacityDao.update(capacity.getId(), capacity);
        }
    }

    class CapacityChecker extends ManagedContextTimerTask {
        @Override
        protected void runInContext() {
            try {
                logger.debug("Running Capacity Checker ... ");
                checkForAlerts();
                logger.debug("Done running Capacity Checker ... ");
            } catch (Throwable t) {
                logger.error("Exception in CapacityChecker", t);
            }
        }
    }

    public void checkForAlerts() {

        recalculateCapacity();

        if (mailSender == null) {
            return;
        }

        //Get all datacenters, pods and clusters in the system.
        List<DataCenterVO> dataCenterList = _dcDao.listAll();
        List<ClusterVO> clusterList = _clusterDao.listAll();
        List<HostPodVO> podList = _podDao.listAll();
        //Get capacity types at different levels
        List<Short> dataCenterCapacityTypes = getCapacityTypesAtZoneLevel();
        List<Short> podCapacityTypes = getCapacityTypesAtPodLevel();
        List<Short> clusterCapacityTypes = getCapacityTypesAtClusterLevel();

        // Generate Alerts for Zone Level capacities
        for (DataCenterVO dc : dataCenterList) {
            for (Short capacityType : dataCenterCapacityTypes) {
                List<SummedCapacity> capacity = _capacityDao.findCapacityBy(capacityType.intValue(), dc.getId(), null, null);

                if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE ||
                    capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE ||
                    capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) {
                    capacity.add(getUsedStats(capacityType, dc.getId(), null, null));
                }
                if (capacity == null || capacity.isEmpty()) {
                    continue;
                }
                double totalCapacity = capacity.get(0).getTotalCapacity();
                double usedCapacity = capacity.get(0).getUsedCapacity();
                if (totalCapacity != 0 && usedCapacity / totalCapacity > _capacityTypeThresholdMap.get(capacityType)) {
                    generateEmailAlert(dc, null, null, totalCapacity, usedCapacity, capacityType);
                }
            }
        }

        // Generate Alerts for Pod Level capacities
        for (HostPodVO pod : podList) {
            for (Short capacityType : podCapacityTypes) {
                List<SummedCapacity> capacity = _capacityDao.findCapacityBy(capacityType.intValue(), pod.getDataCenterId(), pod.getId(), null);
                if (capacity == null || capacity.isEmpty()) {
                    continue;
                }
                double totalCapacity = capacity.get(0).getTotalCapacity();
                double usedCapacity = capacity.get(0).getUsedCapacity();
                if (totalCapacity != 0 && usedCapacity / totalCapacity > _capacityTypeThresholdMap.get(capacityType)) {
                    generateEmailAlert(ApiDBUtils.findZoneById(pod.getDataCenterId()), pod, null, totalCapacity, usedCapacity, capacityType);
                }
            }
        }

        // Generate Alerts for Cluster Level capacities
        for (ClusterVO cluster : clusterList) {
            for (Short capacityType : clusterCapacityTypes) {
                List<SummedCapacity> capacity = _capacityDao.findCapacityBy(capacityType.intValue(), cluster.getDataCenterId(), null, cluster.getId());

                // cpu and memory allocated capacity notification threshold can be defined at cluster level, so getting the value if they are defined at cluster level
                double threshold;
                switch (capacityType) {
                case Capacity.CAPACITY_TYPE_STORAGE:
                    capacity.add(getUsedStats(capacityType, cluster.getDataCenterId(), cluster.getPodId(), cluster.getId()));
                    threshold = StorageCapacityThreshold.valueIn(cluster.getId());
                    break;
                case Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED:
                    threshold = StorageAllocatedCapacityThreshold.valueIn(cluster.getId());
                    break;
                case Capacity.CAPACITY_TYPE_CPU:
                    threshold = CPUCapacityThreshold.valueIn(cluster.getId());
                    break;
                case Capacity.CAPACITY_TYPE_MEMORY:
                    threshold = MemoryCapacityThreshold.valueIn(cluster.getId());
                    break;
                default:
                    threshold = _capacityTypeThresholdMap.get(capacityType);
                }
                if (capacity == null || capacity.isEmpty()) {
                    continue;
                }

                double totalCapacity = capacity.get(0).getTotalCapacity();
                double usedCapacity = capacity.get(0).getUsedCapacity() + capacity.get(0).getReservedCapacity();
                if (totalCapacity != 0 && usedCapacity / totalCapacity > threshold) {
                    generateEmailAlert(ApiDBUtils.findZoneById(cluster.getDataCenterId()), ApiDBUtils.findPodById(cluster.getPodId()), cluster, totalCapacity,
                            usedCapacity, capacityType);
                }
            }
        }

    }

    private SummedCapacity getUsedStats(short capacityType, long zoneId, Long podId, Long clusterId) {
        CapacityVO capacity = null;

        if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) {
            capacity = _storageMgr.getSecondaryStorageUsedStats(null, zoneId);
        } else if (capacityType == Capacity.CAPACITY_TYPE_STORAGE) {
            capacity = _storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId);
        } else if (capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE) {
            capacity = _storageMgr.getObjectStorageUsedStats(zoneId);
        } else if (capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) {
            capacity = (CapacityVO) backupManager.getBackupStorageUsedStats(zoneId);
        }
        if (capacity != null) {
            return new SummedCapacity(capacity.getUsedCapacity(), 0, capacity.getTotalCapacity(), capacityType, clusterId, podId);
        } else {
            return null;
        }
    }

    private void generateEmailAlert(DataCenterVO dc, HostPodVO pod, ClusterVO cluster, double totalCapacity, double usedCapacity, short capacityType) {

        String msgSubject = null;
        String msgContent = null;
        String percentual = formatPercent(usedCapacity / totalCapacity);
        String totalInMB = formatBytesToMegabytes(totalCapacity);
        String usedInMB = formatBytesToMegabytes(usedCapacity);
        String totalInString = String.valueOf(totalCapacity);
        String usedInString = String.valueOf(usedCapacity);
        AlertType alertType = null;
        Long podId = pod == null ? null : pod.getId();
        Long clusterId = cluster == null ? null : cluster.getId();
        String clusterName = cluster == null ? null : cluster.getName();
        String podName = pod == null ? null : pod.getName();
        String dataCenterName = dc.getName();

        switch (capacityType) {
            case Capacity.CAPACITY_TYPE_MEMORY:
                msgSubject = String.format("System Alert: Low Available Memory in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName, dataCenterName);
                msgContent = String.format("System memory is low, total: %s MB, used: %s MB (%s%%).", totalInMB, usedInMB, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_MEMORY;
                break;
            case Capacity.CAPACITY_TYPE_CPU:
                msgSubject = String.format("System Alert: Low Unallocated CPU in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName, dataCenterName);
                msgContent = String.format("Unallocated CPU is low, total: %s Mhz, used: %s Mhz (%s%%).", DfWhole.format(totalCapacity), DfWhole.format(usedCapacity), percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_CPU;
                break;
            case Capacity.CAPACITY_TYPE_STORAGE:
                msgSubject = String.format("System Alert: Low Available Storage in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName, dataCenterName);
                msgContent = String.format("Available storage space is low, total: %s MB, used: %s MB (%s%%).", totalInMB, usedInMB, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_STORAGE;
                break;
            case Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED:
                msgSubject = String.format("System Alert: Remaining unallocated Storage is low in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName,
                        dataCenterName);
                msgContent = String.format("Unallocated storage space is low, total: %s MB, allocated: %s MB (%s%%)", totalInMB, usedInMB, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_STORAGE_ALLOCATED;
                break;
            case Capacity.CAPACITY_TYPE_LOCAL_STORAGE:
                msgSubject = String.format("System Alert: Remaining unallocated Local Storage is low in cluster [%s] pod [%s] of availability zone [%s].", clusterName, podName,
                        dataCenterName);
                msgContent = String.format("Unallocated storage space is low, total: %s MB, allocated: %s MB (%s%%)", totalInMB, usedInMB, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_LOCAL_STORAGE;
                break;
            case Capacity.CAPACITY_TYPE_PRIVATE_IP:
                msgSubject = String.format("System Alert: Number of unallocated private IPs is low in pod %s of availability zone [%s].", podName, dataCenterName);
                msgContent = String.format("Number of unallocated private IPs is low, total: %s, allocated: %s (%s%%)", totalInString, usedInString, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_PRIVATE_IP;
                break;
            case Capacity.CAPACITY_TYPE_SECONDARY_STORAGE:
                msgSubject = String.format("System Alert: Low Available Secondary Storage in availability zone [%s].", dataCenterName);
                msgContent = String.format("Available secondary storage space is low, total: %s MB, used: %s MB (%s%%).", totalInMB, usedInMB, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_SECONDARY_STORAGE;
                break;
            case Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP:
                msgSubject = String.format("System Alert: Number of unallocated virtual network public IPs is low in availability zone [%s].", dataCenterName);
                msgContent = String.format("Number of unallocated public IPs is low, total: %s, allocated: %s (%s%%).", totalInString, usedInString, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_VIRTUAL_NETWORK_PUBLIC_IP;
                break;
            case Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP:
                msgSubject = String.format("System Alert: Number of unallocated shared network IPs is low in availability zone [%s].", dataCenterName);
                msgContent = String.format("Number of unallocated shared network IPs is low, total: %s, allocated: %s (%s%%).", totalInString, usedInString, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_DIRECT_ATTACHED_PUBLIC_IP;
                break;
            case Capacity.CAPACITY_TYPE_VLAN:
                msgSubject = String.format("System Alert: Number of unallocated VLANs is low in availability zone [%s].", dataCenterName);
                msgContent = String.format("Number of unallocated VLANs is low, total: %s, allocated: %s (%s%%).", totalInString, usedInString, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_VLAN;
                break;
            case Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET:
                msgSubject = String.format("System Alert: Number of unallocated virtual network guest IPv6 subnets is low in availability zone [%s].", dc.getName());
                msgContent = String.format("Number of unallocated virtual network guest IPv6 subnets is low, total: [%s], allocated: [%s] (%s%%).", totalInString, usedInString, percentual);
                alertType = AlertManager.AlertType.ALERT_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET;
                break;
            case Capacity.CAPACITY_TYPE_BACKUP_STORAGE:
                msgSubject = "System Alert: Low Available Backup Storage in availability zone " + dc.getName();
                msgContent = "Available backup storage space is low, total: " + totalInString + " MB, used: " + usedInString + " MB (" + percentual + "%)";
                alertType = AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE;
                break;
            case Capacity.CAPACITY_TYPE_OBJECT_STORAGE:
                msgSubject = "System Alert: Low Available Object Storage in availability zone " + dc.getName();
                msgContent = "Available object storage space is low, total: " + totalInString + " MB, used: " + usedInString + " MB (" + percentual + "%)";
                alertType = AlertManager.AlertType.ALERT_TYPE_OBJECT_STORAGE;
                break;
        }

        try {
            logger.debug("Sending alert with subject [{}] and content [{}].", msgSubject, msgContent);
            sendAlert(alertType, dc.getId(), podId, clusterId, msgSubject, msgContent);
        } catch (Exception ex) {
            logger.error("Exception in CapacityChecker", ex);
        }
    }

    private List<Short> getCapacityTypesAtZoneLevel() {

        List<Short> dataCenterCapacityTypes = new ArrayList<>();
        dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP);
        dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP);
        dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE);
        dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VLAN);
        dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET);
        dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_BACKUP_STORAGE);
        dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_OBJECT_STORAGE);
        return dataCenterCapacityTypes;

    }

    private List<Short> getCapacityTypesAtPodLevel() {

        List<Short> podCapacityTypes = new ArrayList<>();
        podCapacityTypes.add(Capacity.CAPACITY_TYPE_PRIVATE_IP);
        return podCapacityTypes;

    }

    private List<Short> getCapacityTypesAtClusterLevel() {

        List<Short> clusterCapacityTypes = new ArrayList<>();
        clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_CPU);
        clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_MEMORY);
        clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_STORAGE);
        clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED);
        clusterCapacityTypes.add(Capacity.CAPACITY_TYPE_LOCAL_STORAGE);
        return clusterCapacityTypes;

    }

    public void clearAlert(short alertType, long dataCenterId, Long podId) {
        if (alertType != -1) {
            AlertVO alert = _alertDao.getLastAlert(alertType, dataCenterId, podId, null);
            if (alert != null) {
                AlertVO updatedAlert = _alertDao.createForUpdate();
                updatedAlert.setResolved(new Date());
                _alertDao.update(alert.getId(), updatedAlert);
            }
        }
    }

    public void sendAlert(AlertType alertType, long dataCenterId, Long podId, Long clusterId, String subject, String content)
            throws MessagingException, UnsupportedEncodingException {
        DataCenterVO zone = _dcDao.findById(dataCenterId);
        HostPodVO pod = podId == null ? null : _podDao.findById(podId);
        ClusterVO cluster = clusterId == null ? null : _clusterDao.findById(clusterId);
        sendAlert(alertType, zone, pod, cluster, subject, content);
    }

    public void sendAlert(AlertType alertType, DataCenter dataCenter, Pod pod, Cluster cluster, String subject, String content)
            throws MessagingException, UnsupportedEncodingException {
        Long clusterId = cluster == null ? null : cluster.getId();
        Long podId = pod == null ? null : pod.getId();
        long dcId = dataCenter == null ? 0L : dataCenter.getId();
        logger.warn(String.format("alertType=[%s] dataCenterId=[%s] podId=[%s] clusterId=[%s] message=[%s].", alertType, dcId, podId, clusterId, subject));
        AlertVO alert = getAlertForTrivialAlertType(alertType, dcId, podId, clusterId);

        if (alert == null) {
            AlertVO newAlert = new AlertVO();
            newAlert.setType(alertType.getType());
            newAlert.setSubject(subject);
            newAlert.setContent(content);
            newAlert.setClusterId(clusterId);
            newAlert.setPodId(podId);
            newAlert.setDataCenterId(dcId);
            newAlert.setSentCount(1);
            newAlert.setLastSent(new Date());
            newAlert.setName(alertType.getName());
            _alertDao.persist(newAlert);
        } else {
            logger.debug("Have already sent: " + alert.getSentCount() + " emails for alert type '" + alertType + "' -- skipping send email");
            return;
        }

        if (ArrayUtils.isEmpty(recipients)) {
            logger.warn(String.format("No recipients set in global setting 'alert.email.addresses', "
                    + "skipping sending alert with subject [%s] and content [%s].", subject, content));
            return;
        }

        SMTPMailProperties mailProps = new SMTPMailProperties();
        mailProps.setSender(new MailAddress(senderAddress));
        mailProps.setSubject(subject);
        mailProps.setContent(content);
        mailProps.setContentType("text/plain");

        Set<MailAddress> addresses = new HashSet<>();
        for (String recipient : recipients) {
            addresses.add(new MailAddress(recipient));
        }

        mailProps.setRecipients(addresses);

        sendMessage(mailProps);

    }

    @Nullable
    private AlertVO getAlertForTrivialAlertType(AlertType alertType, long dataCenterId, Long podId, Long clusterId) {
        AlertVO alert = null;
        if (!ALERTS.contains(alertType)) {
            alert = _alertDao.getLastAlert(alertType.getType(), dataCenterId, podId, clusterId);
        }
        return alert;
    }

    protected void sendMessage(SMTPMailProperties mailProps) {
        _executor.execute(new Runnable() {
            @Override
            public void run() {
                mailSender.sendMail(mailProps);
            }
        });
    }

    private static String formatPercent(double percentage) {
        return DfPct.format(percentage * 100);
    }

    private static String formatBytesToMegabytes(double bytes) {
        double megaBytes = (bytes / (1024 * 1024));
        return DfWhole.format(megaBytes);
    }

    @Override
    public String getConfigComponentName() {
        return AlertManager.class.getSimpleName();
    }

    @Override
    public ConfigKey<?>[] getConfigKeys() {
        return new ConfigKey<?>[] {CPUCapacityThreshold, MemoryCapacityThreshold, StorageAllocatedCapacityThreshold, StorageCapacityThreshold, AlertSmtpEnabledSecurityProtocols,
            AlertSmtpUseStartTLS, Ipv6SubnetCapacityThreshold, AlertSmtpUseAuth};
    }

    @Override
    @ActionEvent(eventType = EventTypes.ALERT_GENERATE, eventDescription = "generating alert", async = true)
    public boolean generateAlert(AlertType alertType, long dataCenterId, Long podId, String msg) {
        try {
            sendAlert(alertType, dataCenterId, podId, msg, msg);
            return true;
        } catch (Exception ex) {
            logger.warn("Failed to generate an alert of type=" + alertType + "; msg=" + msg);
            return false;
        }
    }
}