| #!/usr/bin/env python |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| """ |
| Tests for host control state |
| """ |
| |
| from marvin.cloudstackAPI import (updateHost, updateConfiguration) |
| from nose.plugins.attrib import attr |
| from marvin.cloudstackTestCase import cloudstackTestCase |
| from marvin.lib.common import (get_domain, |
| get_zone, |
| get_template, |
| list_hosts, |
| list_routers, |
| list_ssvms, |
| list_clusters, |
| list_hosts) |
| from marvin.lib.base import (Account, |
| Domain, |
| Host, |
| ServiceOffering, |
| VirtualMachine) |
| from marvin.sshClient import SshClient |
| from marvin.lib.decoratorGenerators import skipTestIf |
| from marvin.lib.utils import wait_until |
| import logging |
| import time |
| |
| |
| class TestHostControlState(cloudstackTestCase): |
| |
| @classmethod |
| def setUpClass(cls): |
| cls.testClient = super(TestHostControlState, cls).getClsTestClient() |
| cls.apiclient = cls.testClient.getApiClient() |
| cls.services = cls.testClient.getParsedTestDataConfig() |
| # Get Zone, Domain and templates |
| cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) |
| cls.hypervisor = cls.testClient.getHypervisorInfo() |
| cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ |
| |
| cls.template = get_template( |
| cls.apiclient, |
| cls.zone.id, |
| cls.hypervisor |
| ) |
| |
| cls.services["virtual_machine"]["zoneid"] = cls.zone.id |
| cls.services["template"] = cls.template.id |
| cls.services["zoneid"] = cls.zone.id |
| |
| cls._cleanup = [] |
| |
| cls.domain = Domain.create( |
| cls.apiclient, |
| cls.services["acl"]["domain1"] |
| ) |
| cls._cleanup.append(cls.domain) |
| cls.account = Account.create( |
| cls.apiclient, |
| cls.services["account"], |
| domainid=cls.domain.id |
| ) |
| cls._cleanup.append(cls.account) |
| cls.service_offering = ServiceOffering.create( |
| cls.apiclient, |
| cls.services["service_offerings"]["tiny"] |
| ) |
| cls._cleanup.append(cls.service_offering) |
| cls.vm = VirtualMachine.create( |
| cls.apiclient, |
| cls.services["virtual_machine"], |
| templateid=cls.template.id, |
| accountid=cls.account.name, |
| domainid=cls.account.domainid, |
| serviceofferingid=cls.service_offering.id |
| ) |
| cls._cleanup.append(cls.vm) |
| |
| @classmethod |
| def tearDownClass(cls): |
| super(TestHostControlState, cls).tearDownClass() |
| |
| def setUp(self): |
| self.apiclient = self.testClient.getApiClient() |
| self.cleanup = [] |
| return |
| |
| def tearDown(self): |
| super(TestHostControlState, self).tearDown() |
| |
| def disable_host(self, id): |
| cmd = updateHost.updateHostCmd() |
| cmd.id = id |
| cmd.allocationstate = "Disable" |
| response = self.apiclient.updateHost(cmd) |
| self.assertEqual(response.resourcestate, "Disabled") |
| |
| def enable_host(self, id): |
| cmd = updateHost.updateHostCmd() |
| cmd.id = id |
| cmd.allocationstate = "Enable" |
| response = self.apiclient.updateHost(cmd) |
| self.assertEqual(response.resourcestate, "Enabled") |
| |
| def get_host_ipaddress(self, hostId): |
| hosts = list_hosts( |
| self.apiclient, |
| type='Routing', |
| id=hostId |
| ) |
| return hosts[0].ipaddress |
| |
| def stop_agent(self, host_ipaddress): |
| SshClient(host_ipaddress, port=22, user=self.hostConfig["username"], passwd=self.hostConfig["password"]).execute\ |
| ("systemctl stop cloudstack-agent || service cloudstack-agent stop") |
| |
| def start_agent(self, host_ipaddress): |
| SshClient(host_ipaddress, port=22, user=self.hostConfig["username"], passwd=self.hostConfig["password"]).execute\ |
| ("systemctl start cloudstack-agent || service cloudstack-agent start") |
| |
| def verify_uservm_host_control_state(self, vm_id, state): |
| list_vms = VirtualMachine.list( |
| self.apiclient, |
| id=vm_id |
| ) |
| vm = list_vms[0] |
| self.assertEqual(vm.hostcontrolstate, |
| state, |
| msg="host control state should be %s, but it is %s" % (state, vm.hostcontrolstate)) |
| |
| def verify_ssvm_host_control_state(self, vm_id, state): |
| list_ssvm_response = list_ssvms( |
| self.apiclient, |
| id=vm_id |
| ) |
| vm = list_ssvm_response[0] |
| self.assertEqual(vm.hostcontrolstate, |
| state, |
| msg="host control state should be %s, but it is %s" % (state, vm.hostcontrolstate)) |
| |
| def verify_router_host_control_state(self, vm_id, state): |
| list_router_response = list_routers( |
| self.apiclient, |
| id=vm_id |
| ) |
| vm = list_router_response[0] |
| self.assertEqual(vm.hostcontrolstate, |
| state, |
| msg="host control state should be %s, but it is %s" % (state, vm.hostcontrolstate)) |
| |
| @attr(tags=["basic", "advanced"], required_hardware="false") |
| def test_uservm_host_control_state(self): |
| """ Verify host control state for user vm """ |
| # 1. verify hostcontrolstate = Enabled |
| # 2. Disable the host, verify hostcontrolstate = Disabled |
| |
| list_vms = VirtualMachine.list( |
| self.apiclient, |
| id=self.vm.id |
| ) |
| host_id = list_vms[0].hostid |
| |
| self.verify_uservm_host_control_state(self.vm.id, "Enabled") |
| |
| self.disable_host(host_id) |
| self.verify_uservm_host_control_state(self.vm.id, "Disabled") |
| |
| if self.hypervisor == "kvm": |
| host_ipaddress = self.get_host_ipaddress(host_id) |
| |
| self.stop_agent(host_ipaddress) |
| time.sleep(5) # wait for the host to be Disconnected |
| self.verify_uservm_host_control_state(self.vm.id, "Offline") |
| |
| self.enable_host(host_id) |
| self.verify_uservm_host_control_state(self.vm.id, "Offline") |
| |
| self.start_agent(host_ipaddress) |
| time.sleep(10) # wait for the host to be Up |
| self.verify_uservm_host_control_state(self.vm.id, "Enabled") |
| |
| else: |
| self.enable_host(host_id) |
| self.verify_uservm_host_control_state(self.vm.id, "Enabled") |
| |
| @attr(tags=["basic", "advanced"], required_hardware="false") |
| def test_ssvm_host_control_state(self): |
| """ Verify host control state for systemvm """ |
| # 1. verify hostcontrolstate = Enabled |
| # 2. Disable the host, verify hostcontrolstate = Disabled |
| |
| list_ssvm_response = list_ssvms( |
| self.apiclient, |
| systemvmtype='secondarystoragevm', |
| state='Running', |
| zoneid=self.zone.id |
| ) |
| self.assertEqual( |
| isinstance(list_ssvm_response, list), |
| True, |
| "Check list response returns a valid list" |
| ) |
| ssvm = list_ssvm_response[0] |
| host_id = ssvm.hostid |
| |
| self.verify_ssvm_host_control_state(ssvm.id, "Enabled") |
| |
| self.disable_host(host_id) |
| self.verify_ssvm_host_control_state(ssvm.id, "Disabled") |
| |
| self.enable_host(host_id) |
| self.verify_ssvm_host_control_state(ssvm.id, "Enabled") |
| |
| @attr(tags=["basic", "advanced"], required_hardware="false") |
| def test_router_host_control_state(self): |
| """ Verify host control state for router """ |
| # 1. verify hostcontrolstate = Enabled |
| # 2. Disable the host, verify hostcontrolstate = Disabled |
| |
| list_router_response = list_routers( |
| self.apiclient, |
| state='Running', |
| listall=True, |
| zoneid=self.zone.id |
| ) |
| self.assertEqual( |
| isinstance(list_router_response, list), |
| True, |
| "Check list response returns a valid list" |
| ) |
| router = list_router_response[0] |
| host_id = router.hostid |
| |
| self.verify_router_host_control_state(router.id, "Enabled") |
| |
| self.disable_host(host_id) |
| self.verify_router_host_control_state(router.id, "Disabled") |
| |
| self.enable_host(host_id) |
| self.verify_router_host_control_state(router.id, "Enabled") |
| |
| |
| class TestAutoEnableDisableHost(cloudstackTestCase): |
| |
| @classmethod |
| def setUpClass(cls): |
| cls.testClient = super(TestAutoEnableDisableHost, cls).getClsTestClient() |
| cls.apiclient = cls.testClient.getApiClient() |
| cls.services = cls.testClient.getParsedTestDataConfig() |
| # Get Zone, Domain and templates |
| cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) |
| cls.hypervisor = cls.testClient.getHypervisorInfo() |
| cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ |
| if cls.hypervisor.lower() not in ['kvm']: |
| cls.hypervisorNotSupported = True |
| return |
| |
| cls.logger = logging.getLogger('TestAutoEnableDisableHost') |
| return |
| |
| @classmethod |
| def tearDownClass(cls): |
| super(TestAutoEnableDisableHost, cls).tearDownClass() |
| |
| def tearDown(self): |
| super(TestAutoEnableDisableHost, self).tearDown() |
| |
| def get_ssh_client(self, ip, username, password, retries=10): |
| """ Setup ssh client connection and return connection """ |
| try: |
| ssh_client = SshClient(ip, 22, username, password, retries) |
| except Exception as e: |
| raise unittest.SkipTest("Unable to create ssh connection: " % e) |
| |
| self.assertIsNotNone( |
| ssh_client, "Failed to setup ssh connection to ip=%s" % ip) |
| |
| return ssh_client |
| |
| def wait_until_host_is_in_state(self, hostid, resourcestate, interval=3, retries=20): |
| def check_resource_state(): |
| response = Host.list( |
| self.apiclient, |
| id=hostid |
| ) |
| if isinstance(response, list): |
| if response[0].resourcestate == resourcestate: |
| self.logger.debug('Host with id %s is in resource state = %s' % (hostid, resourcestate)) |
| return True, None |
| else: |
| self.logger.debug("Waiting for host " + hostid + |
| " to reach state " + resourcestate + |
| ", with current state " + response[0].resourcestate) |
| return False, None |
| |
| done, _ = wait_until(interval, retries, check_resource_state) |
| if not done: |
| raise Exception("Failed to wait for host %s to be on resource state %s" % (hostid, resourcestate)) |
| return True |
| |
| def update_config(self, enable_feature): |
| cmd = updateConfiguration.updateConfigurationCmd() |
| cmd.name = "enable.kvm.host.auto.enable.disable" |
| cmd.value = enable_feature |
| |
| response = self.apiclient.updateConfiguration(cmd) |
| self.debug("updated the parameter %s with value %s" % (response.name, response.value)) |
| |
| def update_health_check_script(self, ip_address, username, password, exit_code): |
| health_check_script_path = "/etc/cloudstack/agent/healthcheck.sh" |
| health_check_agent_property = "agent.health.check.script.path" |
| agent_properties_file_path = "/etc/cloudstack/agent/agent.properties" |
| |
| ssh_client = self.get_ssh_client(ip_address, username, password) |
| ssh_client.execute("echo 'exit %s' > %s" % (exit_code, health_check_script_path)) |
| ssh_client.execute("chmod +x %s" % health_check_script_path) |
| ssh_client.execute("echo '%s=%s' >> %s" % (health_check_agent_property, health_check_script_path, |
| agent_properties_file_path)) |
| ssh_client.execute("service cloudstack-agent restart") |
| |
| def remove_host_health_check(self, ip_address, username, password): |
| health_check_script_path = "/etc/cloudstack/agent/healthcheck.sh" |
| ssh_client = self.get_ssh_client(ip_address, username, password) |
| ssh_client.execute("rm -f %s" % health_check_script_path) |
| |
| def select_host_for_health_checks(self): |
| clusters = list_clusters( |
| self.apiclient, |
| zoneid=self.zone.id |
| ) |
| if not clusters: |
| return None |
| |
| for cluster in clusters: |
| list_hosts_response = list_hosts( |
| self.apiclient, |
| clusterid=cluster.id, |
| type="Routing", |
| resourcestate="Enabled" |
| ) |
| assert isinstance(list_hosts_response, list) |
| if not list_hosts_response or len(list_hosts_response) < 1: |
| continue |
| return list_hosts_response[0] |
| return None |
| |
| def update_host_allocation_state(self, id, enable): |
| cmd = updateHost.updateHostCmd() |
| cmd.id = id |
| cmd.allocationstate = "Enable" if enable else "Disable" |
| response = self.apiclient.updateHost(cmd) |
| self.assertEqual(response.resourcestate, "Enabled" if enable else "Disabled") |
| |
| @attr(tags=["basic", "advanced"], required_hardware="false") |
| @skipTestIf("hypervisorNotSupported") |
| def test_01_auto_enable_disable_kvm_host(self): |
| """Test to auto-enable and auto-disable a KVM host based on health check results |
| |
| # Validate the following: |
| # 1. Enable the KVM Auto Enable/Disable Feature |
| # 2. Set a health check script that fails and observe the host is Disabled |
| # 3. Make the health check script succeed and observe the host is Enabled |
| """ |
| |
| selected_host = self.select_host_for_health_checks() |
| if not selected_host: |
| self.skipTest("Cannot find a KVM host to test the auto-enable-disable feature") |
| |
| username = self.hostConfig["username"] |
| password = self.hostConfig["password"] |
| |
| # Enable the Auto Enable/Disable Configuration |
| self.update_config("true") |
| |
| # Set health check script for failure |
| self.update_health_check_script(selected_host.ipaddress, username, password, 1) |
| self.wait_until_host_is_in_state(selected_host.id, "Disabled", 5, 200) |
| |
| # Set health check script for success |
| self.update_health_check_script(selected_host.ipaddress, username, password, 0) |
| |
| self.wait_until_host_is_in_state(selected_host.id, "Enabled", 5, 200) |
| |
| @attr(tags=["basic", "advanced"], required_hardware="false") |
| @skipTestIf("hypervisorNotSupported") |
| def test_02_disable_host_overrides_auto_enable_kvm_host(self): |
| """Test to override the auto-enabling of a KVM host by an administrator |
| |
| # Validate the following: |
| # 1. Enable the KVM Auto Enable/Disable Feature |
| # 2. Set a health check script that succeeds and observe the host is Enabled |
| # 3. Make the host Disabled |
| # 4. Verify the host does not get auto-enabled after the previous step |
| """ |
| |
| selected_host = self.select_host_for_health_checks() |
| if not selected_host: |
| self.skipTest("Cannot find a KVM host to test the auto-enable-disable feature") |
| |
| username = self.hostConfig["username"] |
| password = self.hostConfig["password"] |
| |
| # Enable the Auto Enable/Disable Configuration |
| self.update_config("true") |
| |
| # Set health check script for failure |
| self.update_health_check_script(selected_host.ipaddress, username, password, 0) |
| self.wait_until_host_is_in_state(selected_host.id, "Enabled", 5, 200) |
| |
| # Manually disable the host |
| self.update_host_allocation_state(selected_host.id, False) |
| |
| # Wait for more than the ping interval |
| time.sleep(70) |
| |
| # Verify the host continues on Disabled state |
| self.wait_until_host_is_in_state(selected_host.id, "Disabled", 5, 200) |
| |
| # Restore the host to Enabled state |
| self.remove_host_health_check(selected_host.ipaddress, username, password) |
| self.update_host_allocation_state(selected_host.id, True) |
| |
| @attr(tags=["basic", "advanced"], required_hardware="false") |
| @skipTestIf("hypervisorNotSupported") |
| def test_03_enable_host_does_not_override_auto_disable_kvm_host(self): |
| """Test to override the auto-disabling of a KVM host by an administrator |
| |
| # Validate the following: |
| # 1. Enable the KVM Auto Enable/Disable Feature |
| # 2. Set a health check script that fails and observe the host is Disabled |
| # 3. Make the host Enabled |
| # 4. Verify the host does get auto-disabled after the previous step |
| """ |
| |
| selected_host = self.select_host_for_health_checks() |
| if not selected_host: |
| self.skipTest("Cannot find a KVM host to test the auto-enable-disable feature") |
| |
| username = self.hostConfig["username"] |
| password = self.hostConfig["password"] |
| |
| # Enable the Auto Enable/Disable Configuration |
| self.update_config("true") |
| |
| # Set health check script for failure |
| self.update_health_check_script(selected_host.ipaddress, username, password, 1) |
| self.wait_until_host_is_in_state(selected_host.id, "Disabled", 5, 200) |
| |
| # Manually enable the host |
| self.update_host_allocation_state(selected_host.id, True) |
| |
| # Verify the host goes back to Disabled state |
| self.wait_until_host_is_in_state(selected_host.id, "Disabled", 5, 200) |
| |
| # Restore the host to Enabled state |
| self.remove_host_health_check(selected_host.ipaddress, username, password) |
| self.update_host_allocation_state(selected_host.id, True) |
