| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| """ BVT tests for Hosts Maintenance | |
| """ | |
| # Import Local Modules | |
| from marvin.cloudstackTestCase import * | |
| from marvin.lib.utils import * | |
| from marvin.lib.base import * | |
| from marvin.lib.common import (get_zone, get_pod, get_suitable_test_template, list_ssvms) | |
| from nose.plugins.attrib import attr | |
| from marvin.lib.decoratorGenerators import skipTestIf | |
| from distutils.util import strtobool | |
| from marvin.sshClient import SshClient | |
| import time | |
| _multiprocess_shared_ = False | |
| MIN_VMS_FOR_TEST = 3 | |
| class TestHostMaintenanceBase(cloudstackTestCase): | |
| def get_ssh_client(self, ip, username, password, retries=10): | |
| """ Setup ssh client connection and return connection """ | |
| try: | |
| ssh_client = SshClient(ip, 22, username, password, retries) | |
| except Exception as e: | |
| raise unittest.SkipTest("Unable to create ssh connection: " % e) | |
| self.assertIsNotNone( | |
| ssh_client, "Failed to setup ssh connection to ip=%s" % ip) | |
| return ssh_client | |
| def wait_until_host_is_in_state(self, hostid, resourcestate, interval=3, retries=20, abort_states=None): | |
| """ | |
| Wait until the host reaches resourcestate. If abort_states is given and the host lands | |
| in one of those states instead, skip the test immediately rather than waiting out the | |
| full timeout: ErrorInPrepareForMaintenance/ErrorInMaintenance while waiting for | |
| "Maintenance" usually means the hypervisor refused to migrate a VM off this host (most | |
| commonly because the hosts in the cluster have incompatible CPUs), which is an | |
| environment limitation, not something this test can exercise meaningfully. | |
| """ | |
| def check_resource_state(): | |
| response = Host.list( | |
| self.apiclient, | |
| id=hostid | |
| ) | |
| if isinstance(response, list): | |
| current_state = response[0].resourcestate | |
| if current_state == resourcestate: | |
| self.logger.debug('Host with id %s is in resource state = %s' % (hostid, resourcestate)) | |
| return True, None | |
| if abort_states and current_state in abort_states: | |
| self.logger.debug('Host with id %s entered abort state = %s' % (hostid, current_state)) | |
| return True, current_state | |
| self.logger.debug("Waiting for host " + hostid + | |
| " to reach state " + resourcestate + | |
| ", with current state " + current_state) | |
| return False, None | |
| done, abort_state = wait_until(interval, retries, check_resource_state) | |
| if abort_state: | |
| raise unittest.SkipTest( | |
| "Host %s entered resource state %s while waiting to reach %s -- the hypervisor " | |
| "rejected the VM migration needed for this test (commonly caused by incompatible " | |
| "CPUs between hosts in the cluster). Skipping this migration-dependent test." | |
| % (hostid, abort_state, resourcestate)) | |
| if not done: | |
| raise Exception("Failed to wait for host %s to be on resource state %s" % (hostid, resourcestate)) | |
| return True | |
| def prepare_host_for_maintenance(self, hostid): | |
| self.logger.debug("Sending Host with id %s to prepareHostForMaintenance" % hostid) | |
| cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() | |
| cmd.id = hostid | |
| response = self.apiclient.prepareHostForMaintenance(cmd) | |
| self.logger.debug("Host with id %s is in prepareHostForMaintenance" % hostid) | |
| self.logger.debug(response) | |
| return response | |
| def cancel_host_maintenance(self, hostid): | |
| self.logger.debug("Canceling Host with id %s from maintain" % hostid) | |
| cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() | |
| cmd.id = hostid | |
| res = self.apiclient.cancelHostMaintenance(cmd) | |
| self.logger.debug("Host with id %s is cancelling maintenance" % hostid) | |
| return res | |
| def revert_host_state_on_failure(self, hostId): | |
| # updateHost(allocationstate=Enable) only has a transition defined from the | |
| # Disabled resource state. If a migration failed while putting the host into | |
| # maintenance, the host is left in one of the maintenance-related error states | |
| # (PrepareForMaintenance, ErrorInPrepareForMaintenance, Maintenance, | |
| # ErrorInMaintenance) instead, and only cancelHostMaintenance (AdminCancelMaintenance) | |
| # can move it back to Enabled from there. Recover via whichever API actually applies. | |
| host = Host.list(self.apiclient, id=hostId)[0] | |
| if host.resourcestate == "Enabled": | |
| return | |
| if host.resourcestate in ("PrepareForMaintenance", "ErrorInPrepareForMaintenance", | |
| "Maintenance", "ErrorInMaintenance"): | |
| self.cancel_host_maintenance(hostId) | |
| self.wait_until_host_is_in_state(hostId, "Enabled", 5, 60) | |
| return | |
| cmd = updateHost.updateHostCmd() | |
| cmd.id = hostId | |
| cmd.allocationstate = "Enable" | |
| response = self.apiclient.updateHost(cmd) | |
| self.assertEqual(response.resourcestate, "Enabled") | |
| class TestHostMaintenance(TestHostMaintenanceBase): | |
| def setUp(self): | |
| self.logger = logging.getLogger('TestHM') | |
| self.stream_handler = logging.StreamHandler() | |
| self.logger.setLevel(logging.DEBUG) | |
| self.logger.addHandler(self.stream_handler) | |
| self.apiclient = self.testClient.getApiClient() | |
| self.hypervisor = self.testClient.getHypervisorInfo() | |
| self.dbclient = self.testClient.getDbConnection() | |
| self.services = self.testClient.getParsedTestDataConfig() | |
| self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) | |
| self.pod = get_pod(self.apiclient, self.zone.id) | |
| self.cleanup = [] | |
| self.hostConfig = self.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ | |
| def tearDown(self): | |
| try: | |
| # Clean up, terminate the created templates | |
| cleanup_resources(self.apiclient, self.cleanup) | |
| except Exception as e: | |
| raise Exception("Warning: Exception during cleanup : %s" % e) | |
| return | |
| def createVMs(self, hostId, number, offering_key="tiny"): | |
| self.template = get_suitable_test_template( | |
| self.apiclient, | |
| self.zone.id, | |
| None, | |
| self.hypervisor | |
| ) | |
| if self.template == FAILED: | |
| assert False, "get_suitable_test_template() failed to return template" | |
| self.logger.debug("Using template %s " % self.template.id) | |
| self.service_offering = ServiceOffering.create( | |
| self.apiclient, | |
| self.services["service_offerings"][offering_key] | |
| ) | |
| self.logger.debug("Using service offering %s " % self.service_offering.id) | |
| self.network_offering = NetworkOffering.create( | |
| self.apiclient, | |
| self.services["l2-network_offering"], | |
| ) | |
| self.network_offering.update(self.apiclient, state='Enabled') | |
| self.services["network"]["networkoffering"] = self.network_offering.id | |
| self.l2_network = Network.create( | |
| self.apiclient, | |
| self.services["l2-network"], | |
| zoneid=self.zone.id, | |
| networkofferingid=self.network_offering.id | |
| ) | |
| vms=[] | |
| for i in range(0, number): | |
| self.services["virtual_machine"]["zoneid"] = self.zone.id | |
| self.services["virtual_machine"]["template"] = self.template.id | |
| self.services["virtual_machine"]["displayname"] = 'vm' + str(i) | |
| self.services["virtual_machine"]["hypervisor"] = self.hypervisor | |
| vm = VirtualMachine.create( | |
| self.apiclient, | |
| self.services["virtual_machine"], | |
| serviceofferingid=self.service_offering.id, | |
| networkids=self.l2_network.id, | |
| hostid=hostId | |
| ) | |
| vms.append(vm) | |
| self.cleanup.append(vm) | |
| self.logger.debug("VM create = {}".format(vm.id)) | |
| self.cleanup.append(self.l2_network) | |
| self.cleanup.append(self.network_offering) | |
| self.cleanup.append(self.service_offering) | |
| return vms | |
| def checkAllVmsRunningOnHost(self, hostId): | |
| listVms1 = VirtualMachine.list( | |
| self.apiclient, | |
| hostid=hostId | |
| ) | |
| if (listVms1 is not None): | |
| self.logger.debug('Vms found to test all running = {} '.format(len(listVms1))) | |
| for vm in listVms1: | |
| if (vm.state != "Running"): | |
| self.logger.debug('VirtualMachine on Host with id = {} is in {}'.format(vm.id, vm.state)) | |
| return (False, None) | |
| response = list_ssvms( | |
| self.apiclient, | |
| hostid=hostId | |
| ) | |
| if isinstance(response, list): | |
| for systemvm in response: | |
| if systemvm.state != 'Running': | |
| self.logger.debug("Found not running VM {}".format(systemvm.name)) | |
| return (False, None) | |
| return (True, None) | |
| def checkVmMigratingOnHost(self, hostId): | |
| vm_migrating=False | |
| listVms1 = VirtualMachine.list( | |
| self.apiclient, | |
| hostid=hostId | |
| ) | |
| if (listVms1 is not None): | |
| self.logger.debug('Vms found = {} '.format(len(listVms1))) | |
| for vm in listVms1: | |
| if (vm.state == "Migrating"): | |
| self.logger.debug('VirtualMachine on Host with id = {} is in {}'.format(vm.id, vm.state)) | |
| vm_migrating=True | |
| break | |
| return (vm_migrating, None) | |
| def migrationsFinished(self, hostId): | |
| migrations_finished=True | |
| listVms1 = VirtualMachine.list( | |
| self.apiclient, | |
| hostid=hostId | |
| ) | |
| if (listVms1 is not None): | |
| numVms = len(listVms1) | |
| migrations_finished = (numVms == 0) | |
| return (migrations_finished, None) | |
| def noOfVMsOnHost(self, hostId): | |
| listVms = VirtualMachine.list( | |
| self.apiclient, | |
| hostid=hostId | |
| ) | |
| no_of_vms=0 | |
| self.logger.debug("Counting VMs on host " + hostId) | |
| if (listVms is not None): | |
| for vm in listVms: | |
| self.logger.debug("VirtualMachine on Host " + hostId + " = " + vm.id) | |
| no_of_vms=no_of_vms+1 | |
| self.logger.debug("Found VMs on host " + str(no_of_vms)) | |
| return no_of_vms | |
| def hostPrepareAndCancelMaintenance(self, target_host_id, other_host_id): | |
| # Wait for all VMs to complete any pending migrations. | |
| if not wait_until(3, 100, self.checkAllVmsRunningOnHost, target_host_id) or \ | |
| not wait_until(3, 100, self.checkAllVmsRunningOnHost, other_host_id): | |
| raise Exception("Failed to wait for all VMs to reach running state to execute test") | |
| self.prepare_host_for_maintenance(target_host_id) | |
| migrations_finished = wait_until(5, 200, self.migrationsFinished, target_host_id) | |
| self.wait_until_host_is_in_state( | |
| target_host_id, "Maintenance", 5, 200, | |
| abort_states=("ErrorInPrepareForMaintenance", "ErrorInMaintenance")) | |
| vm_count_after_maintenance = self.noOfVMsOnHost(target_host_id) | |
| self.cancel_host_maintenance(target_host_id) | |
| self.wait_until_host_is_in_state(target_host_id, "Enabled", 5, 200) | |
| if vm_count_after_maintenance != 0: | |
| self.fail("Host to put to maintenance still has VMs running") | |
| return migrations_finished | |
| @attr( | |
| tags=[ | |
| "advanced", | |
| "advancedns", | |
| "smoke", | |
| "basic", | |
| "eip", | |
| "sg"], | |
| required_hardware="true") | |
| def test_01_cancel_host_maintenace_with_no_migration_jobs(self): | |
| """ | |
| Tests if putting a host with no migrations (0 VMs) work back and forth | |
| 1) Verify if there are at least 2 hosts in enabled state. | |
| 2) Put the host into maintenance verify success | |
| 3) Put the other host into maintenance, verify success | |
| """ | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| hypervisor=self.hypervisor, | |
| resourcestate='Enabled', | |
| state='Up' | |
| ) | |
| for host in listHost: | |
| self.logger.debug('Found Host = {}'.format(host.id)) | |
| if (len(listHost) < 2): | |
| raise unittest.SkipTest("Canceling tests for host maintenance as we need 2 or more hosts up and enabled") | |
| try: | |
| migrations_finished = self.hostPrepareAndCancelMaintenance(listHost[0].id, listHost[1].id) | |
| if migrations_finished: | |
| self.hostPrepareAndCancelMaintenance(listHost[1].id, listHost[0].id) | |
| else: | |
| raise unittest.SkipTest("VMs are still migrating so reverse migration /maintenace skipped") | |
| except unittest.SkipTest: | |
| self.revert_host_state_on_failure(listHost[0].id) | |
| self.revert_host_state_on_failure(listHost[1].id) | |
| raise | |
| except Exception as e: | |
| self.revert_host_state_on_failure(listHost[0].id) | |
| self.revert_host_state_on_failure(listHost[1].id) | |
| self.logger.debug("Exception {}".format(e)) | |
| self.fail("Host maintenance test failed {}".format(str(e))) | |
| @attr( | |
| tags=[ | |
| "advanced", | |
| "advancedns", | |
| "smoke", | |
| "basic", | |
| "eip", | |
| "sg"], | |
| required_hardware="true") | |
| def test_02_cancel_host_maintenace_with_migration_jobs(self): | |
| """ | |
| Tests if putting a host with migrations (3 VMs) work back and forth | |
| 1) Verify if there are at least 2 hosts in enabled state. | |
| 2) Deploy VMs if needed | |
| 3) Put the host into maintenance verify success -ensure existing host has zero running VMs | |
| 4) Put the other host into maintenance, verify success just as step 3 | |
| """ | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| hypervisor=self.hypervisor, | |
| resourcestate='Enabled', | |
| state='Up' | |
| ) | |
| for host in listHost: | |
| self.logger.debug('Found Host = {}'.format(host.id)) | |
| if (len(listHost) < 2): | |
| raise unittest.SkipTest("Canceling tests for host maintenance as we need 2 or more hosts up and enabled") | |
| no_of_vms = self.noOfVMsOnHost(listHost[0].id) | |
| no_of_vms = no_of_vms + self.noOfVMsOnHost(listHost[1].id) | |
| if no_of_vms < MIN_VMS_FOR_TEST: | |
| self.logger.debug("Create VMs as there are not enough vms to check host maintenance") | |
| no_vm_req = MIN_VMS_FOR_TEST - no_of_vms | |
| if (no_vm_req > 0): | |
| self.logger.debug("Creating vms = {}".format(no_vm_req)) | |
| self.vmlist = self.createVMs(listHost[0].id, no_vm_req) | |
| time.sleep(10) | |
| try: | |
| migrations_finished = self.hostPrepareAndCancelMaintenance(listHost[0].id, listHost[1].id) | |
| if migrations_finished: | |
| self.hostPrepareAndCancelMaintenance(listHost[1].id, listHost[0].id) | |
| else: | |
| raise unittest.SkipTest("VMs are still migrating so reverse migration /maintenace skipped") | |
| except unittest.SkipTest: | |
| self.revert_host_state_on_failure(listHost[0].id) | |
| self.revert_host_state_on_failure(listHost[1].id) | |
| raise | |
| except Exception as e: | |
| self.revert_host_state_on_failure(listHost[0].id) | |
| self.revert_host_state_on_failure(listHost[1].id) | |
| self.logger.debug("Exception {}".format(e)) | |
| self.fail("Host maintenance test failed {}".format(str(e))) | |
| @attr( | |
| tags=[ | |
| "advanced", | |
| "advancedns", | |
| "smoke", | |
| "basic", | |
| "eip", | |
| "sg"], | |
| required_hardware="true") | |
| def test_03_cancel_host_maintenace_with_migration_jobs_failure(self): | |
| """ | |
| Tests if putting a host with impossible migrations (2 VMs) work pushes to ErrorInMaintenance state | |
| 1) Verify if there are at least 2 hosts in enabled state. | |
| 2) Tag the host and deploy tagged VMs which cannot be migrated to other host without tags | |
| 3) Put the host into maintenance verify it fails with it reaching ErrorInMaintenance | |
| """ | |
| listHost = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| hypervisor=self.hypervisor, | |
| resourcestate='Enabled', | |
| state='Up' | |
| ) | |
| for host in listHost: | |
| self.logger.debug('Found Host = {}'.format(host.id)) | |
| if (len(listHost) < 2): | |
| raise unittest.SkipTest("Canceling tests for host maintenance as we need 2 or more hosts up and enabled") | |
| target_host_id = listHost[0].id | |
| try: | |
| Host.update(self.apiclient, | |
| id=target_host_id, | |
| hosttags=self.services["service_offerings"]["taggedsmall"]["hosttags"]) | |
| no_of_vms = self.noOfVMsOnHost(target_host_id) | |
| # Need only 2 VMs for this case. | |
| if no_of_vms < 2: | |
| self.logger.debug("Create VMs as there are not enough vms to check host maintenance") | |
| no_vm_req = 2 - no_of_vms | |
| if (no_vm_req > 0): | |
| self.logger.debug("Creating vms = {}".format(no_vm_req)) | |
| self.vmlist = self.createVMs(listHost[0].id, no_vm_req, "taggedsmall") | |
| time.sleep(10) | |
| # Attempt putting host in maintenance and check if ErrorInMaintenance state is reached | |
| self.prepare_host_for_maintenance(target_host_id) | |
| error_in_maintenance_reached = self.wait_until_host_is_in_state(target_host_id, "ErrorInMaintenance", 5, 300) | |
| self.cancel_host_maintenance(target_host_id) | |
| self.wait_until_host_is_in_state(target_host_id, "Enabled", 5, 200) | |
| Host.update(self.apiclient, id=target_host_id, hosttags="") | |
| if not error_in_maintenance_reached: | |
| self.fail("Error in maintenance state should have reached after ports block") | |
| except Exception as e: | |
| self.revert_host_state_on_failure(listHost[0].id) | |
| self.revert_host_state_on_failure(listHost[1].id) | |
| Host.update(self.apiclient, id=target_host_id, hosttags="") | |
| self.logger.debug("Exception {}".format(e)) | |
| self.fail("Host maintenance test failed {}".format(str(e))) | |
| class TestHostMaintenanceAgents(TestHostMaintenanceBase): | |
| @classmethod | |
| def setUpClass(cls): | |
| cls.testClient = super(TestHostMaintenanceAgents, cls).getClsTestClient() | |
| cls.apiclient = cls.testClient.getApiClient() | |
| cls.hypervisor = cls.testClient.getHypervisorInfo() | |
| cls.dbclient = cls.testClient.getDbConnection() | |
| cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) | |
| cls.pod = get_pod(cls.apiclient, cls.zone.id) | |
| cls.services = cls.testClient.getParsedTestDataConfig() | |
| cls.logger = logging.getLogger('TestHMAgents') | |
| cls.stream_handler = logging.StreamHandler() | |
| cls.logger.setLevel(logging.DEBUG) | |
| cls.logger.addHandler(cls.stream_handler) | |
| cls._cleanup = [] | |
| cls.hypervisorNotSupported = False | |
| if cls.hypervisor.lower() not in ['kvm', 'lxc']: | |
| cls.hypervisorNotSupported = True | |
| if not cls.hypervisorNotSupported: | |
| cls.initialsshvalue = cls.is_ssh_enabled() | |
| cls.template = get_suitable_test_template( | |
| cls.apiclient, | |
| cls.zone.id, | |
| None, | |
| cls.hypervisor | |
| ) | |
| if cls.template == FAILED: | |
| assert False, "get_suitable_test_template() failed to return template" | |
| cls.services["virtual_machine"]["zoneid"] = cls.zone.id | |
| cls.services["virtual_machine"]["template"] = cls.template.id | |
| cls.services["virtual_machine"]["hypervisor"] = cls.hypervisor | |
| cls.service_offering = ServiceOffering.create( | |
| cls.apiclient, | |
| cls.services["service_offerings"]["tiny"] | |
| ) | |
| cls._cleanup.append(cls.service_offering) | |
| cls.network_offering = NetworkOffering.create( | |
| cls.apiclient, | |
| cls.services["l2-network_offering"], | |
| ) | |
| cls.network_offering.update(cls.apiclient, state='Enabled') | |
| cls.services["network"]["networkoffering"] = cls.network_offering.id | |
| cls.l2_network = Network.create( | |
| cls.apiclient, | |
| cls.services["l2-network"], | |
| zoneid=cls.zone.id, | |
| networkofferingid=cls.network_offering.id | |
| ) | |
| cls._cleanup.append(cls.l2_network) | |
| cls._cleanup.append(cls.network_offering) | |
| cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ | |
| @classmethod | |
| def tearDownClass(cls): | |
| try: | |
| if not cls.hypervisorNotSupported: | |
| # Revert setting value to the original | |
| cls.set_ssh_enabled(cls.initialsshvalue) | |
| cleanup_resources(cls.apiclient, cls._cleanup) | |
| except Exception as e: | |
| raise Exception("Warning: Exception during cleanup : %s" % e) | |
| def setUp(self): | |
| if not self.hypervisorNotSupported: | |
| self.host = self.get_enabled_host_connected_agent() | |
| self.cleanup = [] | |
| def tearDown(self): | |
| try: | |
| cleanup_resources(self.apiclient, self.cleanup) | |
| except Exception as e: | |
| raise Exception("Warning: Exception during cleanup : %s" % e) | |
| @classmethod | |
| def is_ssh_enabled(cls): | |
| conf = Configurations.list(cls.apiclient, name="kvm.ssh.to.agent") | |
| if not conf: | |
| return False | |
| else: | |
| return bool(strtobool(conf[0].value)) if conf[0].value else False | |
| @classmethod | |
| def updateConfiguration(self, name, value): | |
| cmd = updateConfiguration.updateConfigurationCmd() | |
| cmd.name = name | |
| cmd.value = value | |
| self.apiclient.updateConfiguration(cmd) | |
| @classmethod | |
| def set_ssh_enabled(cls, on): | |
| value = "true" if on else "false" | |
| cls.updateConfiguration('kvm.ssh.to.agent', value) | |
| def wait_until_agent_is_in_state(self, hostid, state, interval=3, retries=20): | |
| def check_agent_state(): | |
| response = Host.list( | |
| self.apiclient, | |
| id=hostid | |
| ) | |
| if isinstance(response, list): | |
| if response[0].state == state: | |
| self.logger.debug('Host agent with id %s is in state = %s' % (hostid, state)) | |
| return True, None | |
| return False, None | |
| done, _ = wait_until(interval, retries, check_agent_state) | |
| if not done: | |
| raise Exception("Failed to wait for host agent %s to be on state %s" % (hostid, state)) | |
| return True | |
| def get_enabled_host_connected_agent(self): | |
| hosts = Host.list( | |
| self.apiclient, | |
| type='Routing', | |
| zoneid=self.zone.id, | |
| podid=self.pod.id, | |
| hypervisor=self.hypervisor, | |
| resourcestate='Enabled', | |
| state='Up' | |
| ) | |
| if len(hosts) < 2: | |
| raise unittest.SkipTest("Host maintenance tests must be tested for 2 or more hosts") | |
| return hosts[0] | |
| def deploy_vm_on_host(self, hostid): | |
| return VirtualMachine.create( | |
| self.apiclient, | |
| self.services["virtual_machine"], | |
| serviceofferingid=self.service_offering.id, | |
| networkids=self.l2_network.id, | |
| hostid=hostid | |
| ) | |
| def assert_host_is_functional_after_cancelling_maintenance(self, hostid): | |
| self.wait_until_agent_is_in_state(hostid, "Up") | |
| self.logger.debug('Deploying VM on host %s' % hostid) | |
| vm = self.deploy_vm_on_host(hostid) | |
| self.assertEqual( | |
| vm.state, | |
| "Running", | |
| "Check VM is running on the host" | |
| ) | |
| self.cleanup.append(vm) | |
| @skipTestIf("hypervisorNotSupported") | |
| @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") | |
| def test_01_cancel_host_maintenance_ssh_enabled_agent_connected(self): | |
| """ | |
| Test cancel maintenance when: 'kvm.ssh.to.agent' = true, agent state = 'Up' | |
| 1) Put host on Maintenance | |
| 2) Cancel maintenance on host | |
| 4) Assert agent is still connected after cancelling maintenance | |
| 3) Deploy VM on the host after cancelling maintenance | |
| """ | |
| if not self.is_ssh_enabled(): | |
| self.set_ssh_enabled(True) | |
| try: | |
| self.prepare_host_for_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Maintenance") | |
| self.cancel_host_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Enabled") | |
| self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) | |
| except Exception as e: | |
| self.revert_host_state_on_failure(self.host.id) | |
| self.fail(e) | |
| @skipTestIf("hypervisorNotSupported") | |
| @attr(tags=["boris", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") | |
| def test_02_cancel_host_maintenance_ssh_enabled_agent_disconnected(self): | |
| """ | |
| Test cancel maintenance when: 'kvm.ssh.to.agent' = true, agent state != 'Up' | |
| 1) Put host on maintenance | |
| 2) SSH into host and stop cloudstack-agent service - host gets Disconnected | |
| 3) Cancel maintenance on host | |
| 4) Assert agent is connected after cancelling maintenance | |
| 5) Deploy VM on the host | |
| """ | |
| if not self.is_ssh_enabled(): | |
| self.set_ssh_enabled(True) | |
| # username, password = self.get_host_credentials(self.host.id) | |
| username = self.hostConfig["username"] | |
| password = self.hostConfig["password"] | |
| try: | |
| self.prepare_host_for_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Maintenance") | |
| ssh_client = self.get_ssh_client(self.host.ipaddress, self.hostConfig["username"], | |
| self.hostConfig["password"]) | |
| ssh_client.execute("service cloudstack-agent stop") | |
| self.wait_until_agent_is_in_state(self.host.id, "Disconnected") | |
| self.cancel_host_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Enabled") | |
| self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) | |
| except Exception as e: | |
| self.revert_host_state_on_failure(self.host.id) | |
| self.fail(e) | |
| @skipTestIf("hypervisorNotSupported") | |
| @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") | |
| def test_03_cancel_host_maintenance_ssh_disabled_agent_connected(self): | |
| """ | |
| Test cancel maintenance when: 'kvm.ssh.to.agent' = false, agent state = 'Up' | |
| 1) Put host on Maintenance | |
| 2) Cancel maintenance on host | |
| 4) Assert agent is still connected after cancelling maintenance | |
| 3) Deploy VM on the host after cancelling maintenance | |
| """ | |
| if self.is_ssh_enabled(): | |
| self.set_ssh_enabled(False) | |
| try: | |
| self.prepare_host_for_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Maintenance") | |
| self.cancel_host_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Enabled") | |
| self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) | |
| except Exception as e: | |
| self.revert_host_state_on_failure(self.host.id) | |
| self.fail(e) | |
| @skipTestIf("hypervisorNotSupported") | |
| @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="true") | |
| def test_04_cancel_host_maintenance_ssh_disabled_agent_disconnected(self): | |
| """ | |
| Test cancel maintenance when: 'kvm.ssh.to.agent' = false, agent state != 'Up' | |
| 1) Put host on maintenance | |
| 2) SSH into host (if possible) and stop cloudstack-agent service - host gets Disconnected. | |
| Skip test if not possible to SSH into host | |
| 3) Cancel maintenance on host - assert cannot cancel maintenance on disconnected host (exception thwown) | |
| 4( SSH into host and start cloudstack-agent service - host gets connected | |
| 5) Cancel maintenance on host | |
| 4) Assert agent is connected after cancelling maintenance | |
| 5) Deploy VM on the host | |
| """ | |
| if self.is_ssh_enabled(): | |
| self.set_ssh_enabled(False) | |
| try: | |
| self.prepare_host_for_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Maintenance") | |
| ssh_client = self.get_ssh_client(self.host.ipaddress, self.hostConfig["username"], | |
| self.hostConfig["password"]) | |
| ssh_client.execute("service cloudstack-agent stop") | |
| self.wait_until_agent_is_in_state(self.host.id, "Disconnected") | |
| except Exception as e: | |
| self.revert_host_state_on_failure(self.host.id) | |
| self.fail(e) | |
| self.assertRaises(Exception, self.cancel_host_maintenance, self.host.id) | |
| try: | |
| ssh_client = self.get_ssh_client(self.host.ipaddress, self.hostConfig["username"], | |
| self.hostConfig["password"]) | |
| ssh_client.execute("service cloudstack-agent start") | |
| self.wait_until_agent_is_in_state(self.host.id, "Up") | |
| self.cancel_host_maintenance(self.host.id) | |
| self.wait_until_host_is_in_state(self.host.id, "Enabled") | |
| self.assert_host_is_functional_after_cancelling_maintenance(self.host.id) | |
| except Exception as e: | |
| self.revert_host_state_on_failure(self.host.id) | |
| self.fail(e) |