// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.ha;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import javax.inject.Inject;

import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.ha.dao.HAConfigDao;
import org.apache.cloudstack.managed.context.ManagedContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;

import com.cloud.agent.AgentManager;
import com.cloud.alert.AlertManager;
import com.cloud.consoleproxy.ConsoleProxyManager;
import com.cloud.dc.ClusterDetailsDao;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.HostPodDao;
import com.cloud.ha.HighAvailabilityManager.Step;
import com.cloud.ha.HighAvailabilityManager.WorkType;
import com.cloud.ha.dao.HighAvailabilityDao;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.network.VpcVirtualNetworkApplianceService;
import com.cloud.resource.ResourceManager;
import com.cloud.server.ManagementServer;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.storage.StorageManager;
import com.cloud.storage.dao.GuestOSCategoryDao;
import com.cloud.storage.dao.GuestOSDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.user.AccountManager;
import com.cloud.vm.UserVmManager;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineManager;
import com.cloud.vm.dao.VMInstanceDao;

@RunWith(MockitoJUnitRunner.class)
public class HighAvailabilityManagerImplTest {
    protected Logger logger = LogManager.getLogger(getClass());
    @Mock
    HighAvailabilityDao _haDao;
    @Mock
    VMInstanceDao _instanceDao;
    @Mock
    HostDao _hostDao;
    @Mock
    DataCenterDao _dcDao;
    @Mock
    HostPodDao _podDao;
    @Mock
    ClusterDetailsDao _clusterDetailsDao;
    @Mock
    ServiceOfferingDao _serviceOfferingDao;
    @Mock
    ManagedContext _managedContext;
    @Mock
    AgentManager _agentMgr;
    @Mock
    AlertManager _alertMgr;
    @Mock
    StorageManager _storageMgr;
    @Mock
    GuestOSDao _guestOSDao;
    @Mock
    GuestOSCategoryDao _guestOSCategoryDao;
    @Mock
    VirtualMachineManager _itMgr;
    @Mock
    AccountManager _accountMgr;
    @Mock
    ResourceManager _resourceMgr;
    @Mock
    ManagementServer _msServer;
    @Mock
    ConfigurationDao _configDao;
    @Mock
    HAConfigDao _haConfigDao;
    @Mock
    VolumeOrchestrationService volumeMgr;
    @Mock
    ConsoleProxyManager consoleProxyManager;
    @Mock
    SecondaryStorageVmManager secondaryStorageVmManager;
    @Mock
    HostVO hostVO;
    @Mock
    VolumeDao volumeDao;
    @Mock
    DataStoreProviderManager dataStoreProviderMgr;
    @Mock
    VpcVirtualNetworkApplianceService routerService;
    @Mock
    UserVmManager userVmManager;

    @Mock
    private HaWorkVO mockWork;

    HighAvailabilityManagerImpl highAvailabilityManager;
    HighAvailabilityManagerImpl highAvailabilityManagerSpy;
    static Method processWorkMethod = null;

    @BeforeClass
    public static void initOnce() {
        try {
            processWorkMethod = HighAvailabilityManagerImpl.class.getDeclaredMethod("processWork", HaWorkVO.class);
            processWorkMethod.setAccessible(true);
        } catch (NoSuchMethodException e) {
        }
    }

    @Before
    public void setup() throws IllegalArgumentException,
            IllegalAccessException, NoSuchFieldException, SecurityException {
        highAvailabilityManager = new HighAvailabilityManagerImpl();
        for (Field injectField : HighAvailabilityManagerImpl.class.getDeclaredFields()) {
            if (injectField.isAnnotationPresent(Inject.class)) {
                injectField.setAccessible(true);
                injectField.set(highAvailabilityManager, this.getClass().getDeclaredField(injectField.getName()).get(this));
            } else if (injectField.getName().equals("_workers")) {
                injectField.setAccessible(true);
                for (Class<?> clz : HighAvailabilityManagerImpl.class.getDeclaredClasses()) {
                    if (clz.getName().equals("com.cloud.ha.HighAvailabilityManagerImpl$WorkerThread")) {
                        Object obj = Array.newInstance(clz, 0);
                        injectField.set(highAvailabilityManager, obj);
                    }
                }
            } else if (injectField.getName().equals("_maxRetries")) {
                injectField.setAccessible(true);
                injectField.set(highAvailabilityManager, 5);
            }
        }
        highAvailabilityManagerSpy = Mockito.spy(highAvailabilityManager);
    }

    @Test
    public void scheduleRestartForVmsOnHost() {
        Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing);
        Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM);
        Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
        Mockito.lenient().when(_instanceDao.listByHostId(42l)).thenReturn(Arrays.asList(Mockito.mock(VMInstanceVO.class)));
        Mockito.when(_podDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(HostPodVO.class));
        Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class));

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);

        highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true, HighAvailabilityManager.ReasonType.HostDown);
    }

    @Test
    public void scheduleRestartForVmsOnHostNotSupported() {
        Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing);
        Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.VMware);

        highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true, HighAvailabilityManager.ReasonType.HostDown);
    }

    @Test
    public void scheduleRestartForVmsOnHostHADisabled() {
        Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing);
        Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM);
        Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);

        highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true, HighAvailabilityManager.ReasonType.HostDown);
    }

    @Test
    public void scheduleRestartForVmsOnHostNonEmptyVMList() {
        Mockito.when(hostVO.getId()).thenReturn(1l);
        Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing);
        Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.XenServer);
        Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
        List<VMInstanceVO> vms = new ArrayList<VMInstanceVO>();
        VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class);
        Mockito.lenient().when(vm1.getHostId()).thenReturn(1l);
        Mockito.when(vm1.getType()).thenReturn(VirtualMachine.Type.User);
        Mockito.when(vm1.isHaEnabled()).thenReturn(true);
        vms.add(vm1);
        VMInstanceVO vm2 = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm2.getHostId()).thenReturn(1l);
        Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.DomainRouter);
        Mockito.when(vm2.isHaEnabled()).thenReturn(true);
        Mockito.when(vm2.getDataCenterId()).thenReturn(1L);
        vms.add(vm2);
        Mockito.when(_instanceDao.listByHostId(Mockito.anyLong())).thenReturn(vms);
        Mockito.when(_instanceDao.findByUuid(vm1.getUuid())).thenReturn(vm1);
        Mockito.when(_instanceDao.findByUuid(vm2.getUuid())).thenReturn(vm2);
        Mockito.when(_podDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(HostPodVO.class));
        Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class));
        Mockito.when(_haDao.findPreviousHA(Mockito.anyLong())).thenReturn(Arrays.asList(Mockito.mock(HaWorkVO.class)));
        Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);

        highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true, HighAvailabilityManager.ReasonType.HostDown);
    }

    @Test
    public void scheduleRestartHADisabled() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getId()).thenReturn(1L);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);

        highAvailabilityManager.scheduleRestart(vm, true);
    }

    @Test
    public void scheduleRestartHostNotSupported() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);
        Mockito.when(vm.getHostId()).thenReturn(1L);
        Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.VMware);

        highAvailabilityManager.scheduleRestart(vm, true);
    }

    @Test
    public void scheduleStop() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getId()).thenReturn(1L);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);
        Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
        Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
        Mockito.when(_haDao.hasBeenScheduled(vm.getId(), WorkType.Stop)).thenReturn(false);
        Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);

        assertTrue(highAvailabilityManager.scheduleStop(vm, 1L, WorkType.Stop));
    }

    @Test
    public void scheduleStopHADisabled() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getId()).thenReturn(1L);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);
        Mockito.when(_haDao.hasBeenScheduled(vm.getId(), WorkType.Stop)).thenReturn(false);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);

        assertFalse(highAvailabilityManager.scheduleStop(vm, 1L, WorkType.Stop));
    }

    @Test
    public void scheduleMigration() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getId()).thenReturn(1L);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);
        Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
        Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
        Mockito.when(vm.getHostId()).thenReturn(1L);
        Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);

        assertTrue(highAvailabilityManager.scheduleMigration(vm));
    }

    @Test
    public void scheduleMigrationHADisabled() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getHostId()).thenReturn(1L);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);

        assertFalse(highAvailabilityManager.scheduleMigration(vm));
    }

    @Test
    public void scheduleDestroy() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getId()).thenReturn(1L);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);
        Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User);
        Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running);
        Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class));

        assertTrue(highAvailabilityManager.scheduleDestroy(vm, 1L, HighAvailabilityManager.ReasonType.HostMaintenance));
    }

    @Test
    public void scheduleDestroyHADisabled() {
        VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vm.getDataCenterId()).thenReturn(1L);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);

        assertFalse(highAvailabilityManager.scheduleDestroy(vm, 1L, HighAvailabilityManager.ReasonType.HostMaintenance));
    }

    @Test
    public void investigateHostStatusSuccess() {
        Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
        Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
        // Set the list of investigators, CheckOnAgentInvestigator suffices for now
        Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class);
        List<Investigator> investigators = new ArrayList<Investigator>();
        investigators.add(investigator);
        highAvailabilityManager.setInvestigators(investigators);
        // Mock isAgentAlive to return host status as Down
        Mockito.when(investigator.getHostAgentStatus(hostVO)).thenReturn(Status.Down);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);

        assertTrue(highAvailabilityManager.investigate(1l) == Status.Down);
    }

    @Test
    public void investigateHostStatusFailure() {
        Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
        Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);
        // Set the list of investigators, CheckOnAgentInvestigator suffices for now
        // Also no need to mock isAgentAlive() as actual implementation returns null
        Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class);
        List<Investigator> investigators = new ArrayList<Investigator>();
        investigators.add(investigator);
        highAvailabilityManager.setInvestigators(investigators);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true);

        assertNull(highAvailabilityManager.investigate(1l));
    }

    @Test
    public void investigateHostStatusHADisabled() {
        Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO);
        Mockito.when(hostVO.getDataCenterId()).thenReturn(1L);

        ConfigKey<Boolean> haEnabled = Mockito.mock(ConfigKey.class);
        highAvailabilityManager.VmHaEnabled = haEnabled;
        Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false);

        assertTrue(highAvailabilityManager.investigate(1L) == Status.Alert);
    }

    private void processWorkWithRetryCount(int count, Step expectedStep) {
        assertNotNull(processWorkMethod);
        HaWorkVO work = new HaWorkVO(1l, VirtualMachine.Type.User, WorkType.Migration, Step.Scheduled, 1l, VirtualMachine.State.Running, count, 12345678l, null);
        Mockito.doReturn(12345678l).when(highAvailabilityManagerSpy).migrate(work);
        try {
            processWorkMethod.invoke(highAvailabilityManagerSpy, work);
        } catch (IllegalAccessException e) {
            logger.info("[ignored] expected IllegalAccessException caught: " + e.getLocalizedMessage());
        } catch (IllegalArgumentException e) {
            logger.info("[ignored] expected IllegalArgumentException caught: " + e.getLocalizedMessage());
        } catch (InvocationTargetException e) {
            logger.info("[ignored] expected InvocationTargetException caught: " + e.getLocalizedMessage());
        }
        assertTrue(work.getStep() == expectedStep);
    }

    @Test
    public void processWorkWithRetryCountExceeded() {
        processWorkWithRetryCount(5, Step.Done); // max retry count is 5
    }

    @Test
    public void processWorkWithRetryCountNotExceeded() {
        processWorkWithRetryCount(3, Step.Scheduled);
    }

    @Test
    public void testCheckAndCancelWorkIfNeeded_Success() {
        Mockito.when(mockWork.getStep()).thenReturn(Step.Investigating);
        Mockito.when(mockWork.getReasonType()).thenReturn(HighAvailabilityManager.ReasonType.HostMaintenance);
        Mockito.when(mockWork.getHostId()).thenReturn(1L);
        Mockito.doReturn(Status.Up).when(highAvailabilityManagerSpy).investigate(1L);
        Mockito.doNothing().when(mockWork).setStep(Step.Cancelled);
        boolean result = highAvailabilityManagerSpy.checkAndCancelWorkIfNeeded(mockWork);
        assertTrue(result);
        Mockito.verify(mockWork).setStep(Step.Cancelled);
    }

    @Test
    public void testCheckAndCancelWorkIfNeeded_StepNotInvestigating() {
        Mockito.when(mockWork.getStep()).thenReturn(Step.Cancelled);
        boolean result = highAvailabilityManagerSpy.checkAndCancelWorkIfNeeded(mockWork);
        assertFalse(result);
        Mockito.verify(mockWork, Mockito.never()).setStep(Mockito.any());
    }

    private void runInvalidReasonCheckAndCancelWorkIfNeeded(HighAvailabilityManager.ReasonType reasonType) {
        Mockito.when(mockWork.getStep()).thenReturn(Step.Investigating);
        Mockito.when(mockWork.getReasonType()).thenReturn(reasonType);
        boolean result = highAvailabilityManagerSpy.checkAndCancelWorkIfNeeded(mockWork);
        assertFalse(result);
        Mockito.verify(mockWork, Mockito.never()).setStep(Mockito.any());
    }

    @Test
    public void testCheckAndCancelWorkIfNeeded_InvalidReasonType() {
        runInvalidReasonCheckAndCancelWorkIfNeeded(HighAvailabilityManager.ReasonType.Unknown);
    }

    @Test
    public void testCheckAndCancelWorkIfNeeded_NullReasonType() {
        runInvalidReasonCheckAndCancelWorkIfNeeded(null);
    }

    @Test
    public void testCheckAndCancelWorkIfNeeded_HostStatusNotUp() {
        Mockito.when(mockWork.getStep()).thenReturn(Step.Investigating);
        Mockito.when(mockWork.getReasonType()).thenReturn(HighAvailabilityManager.ReasonType.HostDown);
        Mockito.when(mockWork.getHostId()).thenReturn(1L);
        Mockito.doReturn(Status.Down).when(highAvailabilityManagerSpy).investigate(1L);
        boolean result = highAvailabilityManagerSpy.checkAndCancelWorkIfNeeded(mockWork);
        assertFalse(result);
        Mockito.verify(mockWork, Mockito.never()).setStep(Mockito.any());
    }
}