| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| """ BVT tests for Hosts Maintenance | |
| """ | |
| from marvin.cloudstackTestCase import * | |
| from marvin.lib.utils import * | |
| from marvin.lib.base import * | |
| from marvin.lib.common import * | |
| from nose.plugins.attrib import attr | |
| _multiprocess_shared_ = False | |
| class TestHostHA(cloudstackTestCase): | |
| hostCountMsg = "Host HA can be tested with at least two hosts, only %s found" | |
| def setUp(self): | |
| self.logger = logging.getLogger('TestHM') | |
| self.stream_handler = logging.StreamHandler() | |
| self.logger.setLevel(logging.DEBUG) | |
| self.logger.addHandler(self.stream_handler) | |
| self.apiclient = self.testClient.getApiClient() | |
| self.hypervisor = self.testClient.getHypervisorInfo() | |
| self.dbclient = self.testClient.getDbConnection() | |
| self.services = self.testClient.getParsedTestDataConfig() | |
| self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) | |
| self.pod = get_pod(self.apiclient, self.zone.id) | |
| self.hostConfig = self.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ | |
| self.cleanup = [] | |
| self.services = { | |
| "service_offering": { | |
| "name": "Ultra Tiny Instance", | |
| "displaytext": "Ultra Tiny Instance", | |
| "cpunumber": 1, | |
| "cpuspeed": 100, | |
| "memory": 128, | |
| }, | |
| "service_offering_local": { | |
| "name": "Ultra Tiny Local Instance", | |
| "displaytext": "Ultra Tiny Local Instance", | |
| "cpunumber": 1, | |
| "cpuspeed": 100, | |
| "memory": 128, | |
| "storagetype": "local" | |
| }, | |
| "vm": { | |
| "username": "root", | |
| "password": "password", | |
| "ssh_port": 22, | |
| # Hypervisor type should be same as | |
| # hypervisor type of cluster | |
| "privateport": 22, | |
| "publicport": 22, | |
| "protocol": 'TCP', | |
| }, | |
| "natrule": { | |
| "privateport": 22, | |
| "publicport": 22, | |
| "startport": 22, | |
| "endport": 22, | |
| "protocol": "TCP", | |
| "cidrlist": '0.0.0.0/0', | |
| }, | |
| "ostype": 'CentOS 5.3 (64-bit)', | |
| "sleep": 60, | |
| "timeout": 10, | |
| } | |
| def tearDown(self): | |
| super(TestHostHA, self).tearDown() | |
| def createVMs(self, hostId, number, local): | |
| self.template = get_template( | |
| self.apiclient, | |
| self.zone.id, | |
| self.services["ostype"] | |
| ) | |
| if self.template == FAILED: | |
| assert False, "get_template() failed to return template with description %s" % self.services["ostype"] | |
| self.logger.debug("Using template %s " % self.template.id) | |
| if local: | |
| self.service_offering = ServiceOffering.create( | |
| self.apiclient, | |
| self.services["service_offering_local"] | |
| ) | |
| else: | |
| self.service_offering = ServiceOffering.create( | |
| self.apiclient, | |
| self.services["service_offering"] | |
| ) | |
| self.logger.debug("Using service offering %s " % self.service_offering.id) | |
| vms = [] | |
| for i in range(0, number): | |
| self.services["vm"]["zoneid"] = self.zone.id | |
| self.services["vm"]["template"] = self.template.id | |
| self.services["vm"]["displayname"] = 'vm' + str(i) | |
| self.services["vm"]["hypervisor"] = self.hypervisor | |
| vm = VirtualMachine.create( | |
| self.apiclient, | |
| self.services["vm"], | |
| serviceofferingid=self.service_offering.id, | |
| hostid=hostId | |
| ) | |
| vms.append(vm) | |
| self.cleanup.append(vm) | |
| self.logger.debug("VM create = {}".format(vm.id)) | |
| return vm | |
| def noOfVMsOnHost(self, hostId): | |
| listVms = VirtualMachine.list( | |
| self.apiclient, | |
| hostid=hostId | |
| ) | |
| vmnos = 0 | |
| if (listVms is not None): | |
| for vm in listVms: | |
| self.logger.debug('VirtualMachine on Hyp 1 = {}'.format(vm.id)) | |
| vmnos = vmnos + 1 | |
| return vmnos | |
| def checkHostDown(self, fromHostIp, testHostIp): | |
| try: | |
| ssh = SshClient(fromHostIp, 22, self.hostConfig["username"], | |
| self.hostConfig["password"]) | |
| res = ssh.execute("ping -c 1 %s" % testHostIp) | |
| result = str(res) | |
| if result.count("100% packet loss") == 1: | |
| return True, 1 | |
| else: | |
| return False, 1 | |
| except Exception as e: | |
| self.logger.debug("Got exception %s" % e) | |
| return False, 1 | |
| def checkHostUp(self, fromHostIp, testHostIp): | |
| try: | |
| ssh = SshClient(fromHostIp, 22, self.hostConfig["username"], | |
| self.hostConfig["password"]) | |
| res = ssh.execute("ping -c 1 %s" % testHostIp) | |
| result = str(res) | |
| if result.count(" 0% packet loss") == 1: | |
| return True, 1 | |
| else: | |
| return False, 1 | |
| except Exception as e: | |
| self.logger.debug("Got exception %s" % e) | |
| return False, 1 | |
| def isOnlyNFSStorageAvailable(self): | |
| if self.zone.localstorageenabled: | |
| return False | |
| storage_pools = StoragePool.list( | |
| self.apiclient, | |
| zoneid=self.zone.id, | |
| listall=True | |
| ) | |
| self.assertEqual( | |
| isinstance(storage_pools, list), | |
| True, | |
| "Check if listStoragePools returns a valid response" | |
| ) | |
| for storage_pool in storage_pools: | |
| if storage_pool.type == 'NetworkFilesystem': | |
| return True | |
| return False | |
| def isOnlyLocalStorageAvailable(self): | |
| if not(self.zone.localstorageenabled): | |
| return False | |
| storage_pools = StoragePool.list( | |
| self.apiclient, | |
| zoneid=self.zone.id, | |
| listall=True | |
| ) | |
| self.assertEqual( | |
| isinstance(storage_pools, list), | |
| True, | |
| "Check if listStoragePools returns a valid response" | |
| ) | |
| for storage_pool in storage_pools: | |
| if storage_pool.type == 'NetworkFilesystem': | |
| return False | |
| return True | |
| def isLocalAndNFSStorageAvailable(self): | |
| if not(self.zone.localstorageenabled): | |
| return False | |
| storage_pools = StoragePool.list( | |
| self.apiclient, | |
| zoneid=self.zone.id, | |
| listall=True | |
| ) | |
| self.assertEqual( | |
| isinstance(storage_pools, list), | |
| True, | |
| "Check if listStoragePools returns a valid response" | |
| ) | |
| for storage_pool in storage_pools: | |
| if storage_pool.type == 'NetworkFilesystem': | |
| return True | |
| return False | |
| def checkHostStateInCloudstack(self, state, hostId): | |
| try: | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| id=hostId | |
| ) | |
| self.assertEqual( | |
| isinstance(listHost, list), | |
| True, | |
| "Check if listHost returns a valid response" | |
| ) | |
| self.assertEqual( | |
| len(listHost), | |
| 1, | |
| "Check if listHost returns a host" | |
| ) | |
| self.logger.debug(" Host state is %s " % listHost[0].state) | |
| if listHost[0].state == state: | |
| return True, 1 | |
| else: | |
| return False, 1 | |
| except Exception as e: | |
| self.logger.debug("Got exception %s" % e) | |
| return False, 1 | |
| def disconnectHostfromNetwork(self, hostIp, timeout): | |
| srcFile = os.path.dirname(os.path.realpath(__file__)) + "/test_host_ha.sh" | |
| if not(os.path.isfile(srcFile)): | |
| self.logger.debug("File %s not found" % srcFile) | |
| raise unittest.SkipTest("Script file %s required for HA not found" % srcFile) | |
| ssh = SshClient(hostIp, 22, self.hostConfig["username"], | |
| self.hostConfig["password"]) | |
| ssh.scp(srcFile, "/root/test_host_ha.sh") | |
| ssh.execute("nohup sh /root/test_host_ha.sh -t %s -d all > /dev/null 2>&1 &\n" % timeout) | |
| return | |
| def stopAgentOnHost(self, hostIp, timeout): | |
| srcFile = os.path.dirname(os.path.realpath(__file__)) + "/test_host_ha.sh" | |
| if not(os.path.isfile(srcFile)): | |
| self.logger.debug("File %s not found" % srcFile) | |
| raise unittest.SkipTest("Script file %s required for HA not found" % srcFile) | |
| ssh = SshClient(hostIp, 22, self.hostConfig["username"], self.hostConfig["password"]) | |
| ssh.scp(srcFile, "/root/test_host_ha.sh") | |
| ssh.execute("nohup sh /root/test_host_ha.sh -t %s -d agent > /dev/null 2>&1 &\n" % timeout) | |
| return | |
| @attr( | |
| tags=[ | |
| "advanced", | |
| "advancedns", | |
| "smoke", | |
| "basic", | |
| "eip", | |
| "sg"], | |
| required_hardware="true") | |
| def test_01_host_ha_with_nfs_storagepool_with_vm(self): | |
| if not(self.isOnlyNFSStorageAvailable()): | |
| raise unittest.SkipTest("Skipping this test as this is for NFS store only.") | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| ) | |
| for host in listHost: | |
| self.logger.debug('Hypervisor = {}'.format(host.id)) | |
| if len(listHost) < 2: | |
| self.logger.debug(self.hostCountMsg % len(listHost)) | |
| raise unittest.SkipTest(self.hostCountMsg % len(listHost)) | |
| no_of_vms = self.noOfVMsOnHost(listHost[0].id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(listHost[1].id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| if no_of_vms < 5: | |
| self.logger.debug("test_01: Create VMs as there are not enough vms to check host ha") | |
| no_vm_req = 5 - no_of_vms | |
| if (no_vm_req > 0): | |
| self.logger.debug("Creating vms = {}".format(no_vm_req)) | |
| self.vmlist = self.createVMs(listHost[0].id, no_vm_req, False) | |
| ha_host = listHost[1] | |
| other_host = listHost[0] | |
| if self.noOfVMsOnHost(listHost[0].id) > self.noOfVMsOnHost(listHost[1].id): | |
| ha_host = listHost[0] | |
| other_host = listHost[1] | |
| self.disconnectHostfromNetwork(ha_host.ipaddress, 400) | |
| hostDown = wait_until(10, 10, self.checkHostDown, other_host.ipaddress, ha_host.ipaddress) | |
| if not(hostDown): | |
| raise unittest.SkipTest("Host %s is not down, cannot proceed with test" % (ha_host.ipaddress)) | |
| hostDownInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Down", ha_host.id) | |
| #the test could have failed here but we will try our best to get host back in consistent state | |
| no_of_vms = self.noOfVMsOnHost(ha_host.id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(other_host.id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| # | |
| hostUp = wait_until(10, 10, self.checkHostUp, other_host.ipaddress, ha_host.ipaddress) | |
| if not(hostUp): | |
| self.logger.debug("Host is down %s, though HA went fine, the environment is not consistent " % (ha_host.ipaddress)) | |
| hostUpInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Up", ha_host.id) | |
| if not(hostDownInCloudstack): | |
| raise self.fail("Host is not down %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| if not(hostUpInCloudstack): | |
| raise self.fail("Host is not up %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| return | |
| @attr( | |
| tags=[ | |
| "advanced", | |
| "advancedns", | |
| "smoke", | |
| "basic", | |
| "eip", | |
| "sg"], | |
| required_hardware="true") | |
| def test_02_host_ha_with_local_storage_and_nfs(self): | |
| if not(self.isLocalAndNFSStorageAvailable()): | |
| raise unittest.SkipTest("Skipping this test as this is for Local storage and NFS storage only.") | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| ) | |
| for host in listHost: | |
| self.logger.debug('Hypervisor = {}'.format(host.id)) | |
| if len(listHost) < 2: | |
| self.logger.debug(self.hostCountMsg % len(listHost)) | |
| raise unittest.SkipTest(self.hostCountMsg % len(listHost)) | |
| no_of_vms = self.noOfVMsOnHost(listHost[0].id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(listHost[1].id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| if no_of_vms < 5: | |
| self.logger.debug("test_02: Create VMs as there are not enough vms to check host ha") | |
| no_vm_req = 5 - no_of_vms | |
| if (no_vm_req > 0): | |
| self.logger.debug("Creating vms = {}".format(no_vm_req)) | |
| self.vmlist = self.createVMs(listHost[0].id, no_vm_req, True) | |
| ha_host = listHost[1] | |
| other_host = listHost[0] | |
| if self.noOfVMsOnHost(listHost[0].id) > self.noOfVMsOnHost(listHost[1].id): | |
| ha_host = listHost[0] | |
| other_host = listHost[1] | |
| self.disconnectHostfromNetwork(ha_host.ipaddress, 400) | |
| hostDown = wait_until(10, 10, self.checkHostDown, other_host.ipaddress, ha_host.ipaddress) | |
| if not(hostDown): | |
| raise unittest.SkipTest("Host %s is not down, cannot proceed with test" % (ha_host.ipaddress)) | |
| hostDownInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Down", ha_host.id) | |
| #the test could have failed here but we will try our best to get host back in consistent state | |
| no_of_vms = self.noOfVMsOnHost(ha_host.id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(other_host.id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| # | |
| hostUp = wait_until(10, 10, self.checkHostUp, other_host.ipaddress, ha_host.ipaddress) | |
| if not(hostUp): | |
| self.logger.debug("Host is down %s, though HA went fine, the environment is not consistent " % (ha_host.ipaddress)) | |
| hostUpInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Up", ha_host.id) | |
| if not(hostDownInCloudstack): | |
| raise self.fail("Host is not down %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| if not(hostUpInCloudstack): | |
| raise self.fail("Host is not up %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| return | |
| @attr( | |
| tags=[ | |
| "advanced", | |
| "advancedns", | |
| "smoke", | |
| "basic", | |
| "eip", | |
| "sg"], | |
| required_hardware="true") | |
| def test_03_host_ha_with_only_local_storage(self): | |
| if not(self.isOnlyLocalStorageAvailable()): | |
| raise unittest.SkipTest("Skipping this test as this is for Local storage only.") | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| ) | |
| for host in listHost: | |
| self.logger.debug('Hypervisor = {}'.format(host.id)) | |
| if len(listHost) < 2: | |
| self.logger.debug(self.hostCountMsg % len(listHost)) | |
| raise unittest.SkipTest(self.hostCountMsg % len(listHost)) | |
| no_of_vms = self.noOfVMsOnHost(listHost[0].id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(listHost[1].id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| if no_of_vms < 5: | |
| self.logger.debug("test_03: Create VMs as there are not enough vms to check host ha") | |
| no_vm_req = 5 - no_of_vms | |
| if (no_vm_req > 0): | |
| self.logger.debug("Creating vms = {}".format(no_vm_req)) | |
| self.vmlist = self.createVMs(listHost[0].id, no_vm_req, True) | |
| ha_host = listHost[1] | |
| other_host = listHost[0] | |
| if self.noOfVMsOnHost(listHost[0].id) > self.noOfVMsOnHost(listHost[1].id): | |
| ha_host = listHost[0] | |
| other_host = listHost[1] | |
| self.disconnectHostfromNetwork(ha_host.ipaddress, 400) | |
| hostDown = wait_until(10, 10, self.checkHostDown, other_host.ipaddress, ha_host.ipaddress) | |
| if not(hostDown): | |
| raise unittest.SkipTest("Host %s is not down, cannot proceed with test" % (ha_host.ipaddress)) | |
| hostDownInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Alert", ha_host.id) | |
| #the test could have failed here but we will try our best to get host back in consistent state | |
| no_of_vms = self.noOfVMsOnHost(ha_host.id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(other_host.id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| # | |
| hostUp = wait_until(10, 10, self.checkHostUp, other_host.ipaddress, ha_host.ipaddress) | |
| if not(hostUp): | |
| self.logger.debug("Host is down %s, though HA went fine, the environment is not consistent " % (ha_host.ipaddress)) | |
| hostUpInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Up", ha_host.id) | |
| if not(hostDownInCloudstack): | |
| raise self.fail("Host is not in alert %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| if not(hostUpInCloudstack): | |
| raise self.fail("Host is not up %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| return | |
| @attr( | |
| tags=[ | |
| "advanced", | |
| "advancedns", | |
| "smoke", | |
| "basic", | |
| "eip", | |
| "sg"], | |
| required_hardware="true") | |
| def test_04_host_ha_vmactivity_check(self): | |
| if not(self.isOnlyNFSStorageAvailable()): | |
| raise unittest.SkipTest("Skipping this test as this is for NFS store only.") | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| ) | |
| for host in listHost: | |
| self.logger.debug('Hypervisor = {}'.format(host.id)) | |
| if len(listHost) < 2: | |
| self.logger.debug(self.hostCountMsg % len(listHost)) | |
| raise unittest.SkipTest(self.hostCountMsg % len(listHost)) | |
| no_of_vms = self.noOfVMsOnHost(listHost[0].id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(listHost[1].id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| if no_of_vms < 5: | |
| self.logger.debug("test_01: Create VMs as there are not enough vms to check host ha") | |
| no_vm_req = 5 - no_of_vms | |
| if (no_vm_req > 0): | |
| self.logger.debug("Creating vms = {}".format(no_vm_req)) | |
| self.vmlist = self.createVMs(listHost[0].id, no_vm_req, False) | |
| ha_host = listHost[1] | |
| other_host = listHost[0] | |
| if self.noOfVMsOnHost(listHost[0].id) > self.noOfVMsOnHost(listHost[1].id): | |
| ha_host = listHost[0] | |
| other_host = listHost[1] | |
| self.stopAgentOnHost(ha_host.ipaddress, 150) | |
| hostDisconnectedInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Disconnected", ha_host.id) | |
| #the test could have failed here but we will try our best to get host back in consistent state | |
| no_of_vms = self.noOfVMsOnHost(ha_host.id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(other_host.id) | |
| self.logger.debug("Number of VMS on hosts = %s" % no_of_vms) | |
| # | |
| hostUpInCloudstack = wait_until(40, 10, self.checkHostStateInCloudstack, "Up", ha_host.id) | |
| if not(hostDisconnectedInCloudstack): | |
| raise self.fail("Host is not disconnected %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| if not(hostUpInCloudstack): | |
| raise self.fail("Host is not up %s, in cloudstack so failing test " % (ha_host.ipaddress)) | |
| return |