test: fix smoke test failures on xen/vmware (#13628) * test: fix smoke test failures on xen/vmware * test: sleep 10 seconds before attaching volume in test_events_resource.py * test: add more time.sleep(10) * test: add more time.sleep(10) in test_vm_strict_host_tags.py * test: fix test_usage.py * test: move time.sleep(10) in test_host_maintenance.py * VMware: fix import DC * test: retry domain deletion in tearDown of test_deploy_vms_in_parallel.py * test: fix tests failures caused by others in drs and host maintenance
diff --git a/test/integration/smoke/test_cluster_drs.py b/test/integration/smoke/test_cluster_drs.py index 53abe7f..42d71b8 100644 --- a/test/integration/smoke/test_cluster_drs.py +++ b/test/integration/smoke/test_cluster_drs.py
@@ -128,13 +128,13 @@ responseS = cls.apiclient.listSystemVms(cmds) if isinstance(responseS, Iterable): for svm in responseS: - if svm.hostid != cls.hosts[0].id: + if svm.hostid != cls.hosts[0].id and svm.state == 'Running': systemVmIds.append(svm.id) cmdv = listRouters.listRoutersCmd() responseR = cls.apiclient.listRouters(cmdv) if isinstance(responseR, Iterable): for svm in responseR: - if svm.hostid != cls.hosts[0].id: + if svm.hostid != cls.hosts[0].id and svm.state == 'Running': systemVmIds.append(svm.id) numToMigrate = len(systemVmIds) cls.logger.debug(f'system vms and routers to migrate -- {numToMigrate}') @@ -142,8 +142,15 @@ cmdM.hostId=cls.hosts[0].id for id in systemVmIds: cmdM.virtualmachineid=id - responseM = cls.apiclient.migrateSystemVm(cmdM) - cls.logger.debug(f'migrated {responseM}') + try: + responseM = cls.apiclient.migrateSystemVm(cmdM) + cls.logger.debug(f'migrated {responseM}') + except Exception as e: + # A system vm/router may have moved out of Running state (e.g. redundant + # router failover, or an in-progress restart from a previous test's cleanup) + # between the listSystemVms/listRouters call above and this migration attempt. + # Don't fail the whole test class setup for one such VM. + cls.logger.debug(f'Skipping migration of {id}, it may no longer be in Running state: {e}') @classmethod
diff --git a/test/integration/smoke/test_deploy_vms_in_parallel.py b/test/integration/smoke/test_deploy_vms_in_parallel.py index 03049a2..6e977bb 100644 --- a/test/integration/smoke/test_deploy_vms_in_parallel.py +++ b/test/integration/smoke/test_deploy_vms_in_parallel.py
@@ -18,6 +18,8 @@ """ P1 for Deploy VM from ISO """ # Import Local Modules +import time + from nose.plugins.attrib import attr from marvin.cloudstackTestCase import cloudstackTestCase from marvin.lib.base import (Account, @@ -123,6 +125,40 @@ ) def tearDown(self): + # Deleting the account only soft-deletes it; the account row (and + # its "needs cleanup" state) is purged asynchronously by the + # account.cleanup.interval background task. Deleting the domain + # right after the account can therefore race with that task and + # fail with "Can't delete the domain yet because it has N + # accounts to cleanup". Delete the account first, then retry the + # domain deletion for a bit to ride out that race. + try: + self.cleanup_resources(self.apiclient, [self.account]) + except Exception as e: + self.debug("Warning: Exception during account cleanup : %s" % e) + + retries_left = 15 + while True: + try: + self.domain.delete(self.apiclient) + break + except Exception as e: + retries_left -= 1 + if "accounts to cleanup" not in str(e): + raise Exception("Warning: Exception during cleanup : %s" % e) + if retries_left <= 0: + # The account cleanup task can get permanently stuck detaching + # the account's data volume from an already-expunged VM (a + # known server-side race between VM expunge and account + # cleanup, unrelated to what this test verifies). Don't fail + # the test on that; just leave the domain/account behind for + # cleanup to retry indefinitely, and log it for visibility. + self.debug("Warning: giving up on domain cleanup, leaving it " + "behind for a later cleanup attempt: %s" % e) + break + time.sleep(5) + + self.cleanup = [] super(TestDeployVMsInParallel, self).tearDown() @attr(
diff --git a/test/integration/smoke/test_events_resource.py b/test/integration/smoke/test_events_resource.py index dac95f8..fce0e6d 100644 --- a/test/integration/smoke/test_events_resource.py +++ b/test/integration/smoke/test_events_resource.py
@@ -147,6 +147,7 @@ diskofferingid=self.disk_offering.id ) self.cleanup.append(volume) + time.sleep(10) virtual_machine.attach_volume( self.apiclient, volume
diff --git a/test/integration/smoke/test_host_maintenance.py b/test/integration/smoke/test_host_maintenance.py index 9a84f32..4b11d8e 100644 --- a/test/integration/smoke/test_host_maintenance.py +++ b/test/integration/smoke/test_host_maintenance.py
@@ -27,6 +27,8 @@ from distutils.util import strtobool from marvin.sshClient import SshClient +import time + _multiprocess_shared_ = False MIN_VMS_FOR_TEST = 3 @@ -43,23 +45,40 @@ return ssh_client - def wait_until_host_is_in_state(self, hostid, resourcestate, interval=3, retries=20): + def wait_until_host_is_in_state(self, hostid, resourcestate, interval=3, retries=20, abort_states=None): + """ + Wait until the host reaches resourcestate. If abort_states is given and the host lands + in one of those states instead, skip the test immediately rather than waiting out the + full timeout: ErrorInPrepareForMaintenance/ErrorInMaintenance while waiting for + "Maintenance" usually means the hypervisor refused to migrate a VM off this host (most + commonly because the hosts in the cluster have incompatible CPUs), which is an + environment limitation, not something this test can exercise meaningfully. + """ def check_resource_state(): response = Host.list( self.apiclient, id=hostid ) if isinstance(response, list): - if response[0].resourcestate == resourcestate: + current_state = response[0].resourcestate + if current_state == resourcestate: self.logger.debug('Host with id %s is in resource state = %s' % (hostid, resourcestate)) return True, None - else: - self.logger.debug("Waiting for host " + hostid + - " to reach state " + resourcestate + - ", with current state " + response[0].resourcestate) + if abort_states and current_state in abort_states: + self.logger.debug('Host with id %s entered abort state = %s' % (hostid, current_state)) + return True, current_state + self.logger.debug("Waiting for host " + hostid + + " to reach state " + resourcestate + + ", with current state " + current_state) return False, None - done, _ = wait_until(interval, retries, check_resource_state) + done, abort_state = wait_until(interval, retries, check_resource_state) + if abort_state: + raise unittest.SkipTest( + "Host %s entered resource state %s while waiting to reach %s -- the hypervisor " + "rejected the VM migration needed for this test (commonly caused by incompatible " + "CPUs between hosts in the cluster). Skipping this migration-dependent test." + % (hostid, abort_state, resourcestate)) if not done: raise Exception("Failed to wait for host %s to be on resource state %s" % (hostid, resourcestate)) return True @@ -82,6 +101,20 @@ return res def revert_host_state_on_failure(self, hostId): + # updateHost(allocationstate=Enable) only has a transition defined from the + # Disabled resource state. If a migration failed while putting the host into + # maintenance, the host is left in one of the maintenance-related error states + # (PrepareForMaintenance, ErrorInPrepareForMaintenance, Maintenance, + # ErrorInMaintenance) instead, and only cancelHostMaintenance (AdminCancelMaintenance) + # can move it back to Enabled from there. Recover via whichever API actually applies. + host = Host.list(self.apiclient, id=hostId)[0] + if host.resourcestate == "Enabled": + return + if host.resourcestate in ("PrepareForMaintenance", "ErrorInPrepareForMaintenance", + "Maintenance", "ErrorInMaintenance"): + self.cancel_host_maintenance(hostId) + self.wait_until_host_is_in_state(hostId, "Enabled", 5, 60) + return cmd = updateHost.updateHostCmd() cmd.id = hostId cmd.allocationstate = "Enable" @@ -245,7 +278,9 @@ self.prepare_host_for_maintenance(target_host_id) migrations_finished = wait_until(5, 200, self.migrationsFinished, target_host_id) - self.wait_until_host_is_in_state(target_host_id, "Maintenance", 5, 200) + self.wait_until_host_is_in_state( + target_host_id, "Maintenance", 5, 200, + abort_states=("ErrorInPrepareForMaintenance", "ErrorInMaintenance")) vm_count_after_maintenance = self.noOfVMsOnHost(target_host_id) @@ -299,11 +334,15 @@ else: raise unittest.SkipTest("VMs are still migrating so reverse migration /maintenace skipped") + except unittest.SkipTest: + self.revert_host_state_on_failure(listHost[0].id) + self.revert_host_state_on_failure(listHost[1].id) + raise except Exception as e: self.revert_host_state_on_failure(listHost[0].id) self.revert_host_state_on_failure(listHost[1].id) self.logger.debug("Exception {}".format(e)) - self.fail("Host maintenance test failed {}".format(e[0])) + self.fail("Host maintenance test failed {}".format(str(e))) @attr( @@ -350,6 +389,8 @@ self.logger.debug("Creating vms = {}".format(no_vm_req)) self.vmlist = self.createVMs(listHost[0].id, no_vm_req) + time.sleep(10) + try: migrations_finished = self.hostPrepareAndCancelMaintenance(listHost[0].id, listHost[1].id) @@ -358,11 +399,15 @@ else: raise unittest.SkipTest("VMs are still migrating so reverse migration /maintenace skipped") + except unittest.SkipTest: + self.revert_host_state_on_failure(listHost[0].id) + self.revert_host_state_on_failure(listHost[1].id) + raise except Exception as e: self.revert_host_state_on_failure(listHost[0].id) self.revert_host_state_on_failure(listHost[1].id) self.logger.debug("Exception {}".format(e)) - self.fail("Host maintenance test failed {}".format(e[0])) + self.fail("Host maintenance test failed {}".format(str(e))) @attr( tags=[ @@ -414,6 +459,8 @@ self.logger.debug("Creating vms = {}".format(no_vm_req)) self.vmlist = self.createVMs(listHost[0].id, no_vm_req, "taggedsmall") + time.sleep(10) + # Attempt putting host in maintenance and check if ErrorInMaintenance state is reached self.prepare_host_for_maintenance(target_host_id) error_in_maintenance_reached = self.wait_until_host_is_in_state(target_host_id, "ErrorInMaintenance", 5, 300) @@ -431,7 +478,7 @@ self.revert_host_state_on_failure(listHost[1].id) Host.update(self.apiclient, id=target_host_id, hosttags="") self.logger.debug("Exception {}".format(e)) - self.fail("Host maintenance test failed {}".format(e[0])) + self.fail("Host maintenance test failed {}".format(str(e))) class TestHostMaintenanceAgents(TestHostMaintenanceBase):
diff --git a/test/integration/smoke/test_kms_lifecycle.py b/test/integration/smoke/test_kms_lifecycle.py index 79deb5b..783a2af 100644 --- a/test/integration/smoke/test_kms_lifecycle.py +++ b/test/integration/smoke/test_kms_lifecycle.py
@@ -65,6 +65,7 @@ cls.apiclient = cls.test_client.getApiClient() cls.zone = get_zone(cls.apiclient, cls.test_client.getZoneForTests()) cls.domain = get_domain(cls.apiclient) + cls.hypervisor = cls.test_client.getHypervisorInfo() cls._cleanup = [] @@ -424,6 +425,10 @@ Test: deploy a VM with its root disk encrypted using a KMS key. Verify that the VM starts and the root volume has the KMS key ID. """ + + if self.hypervisor.lower() != 'kvm': + raise self.skipTest("Skipping test case for non-kvm hypervisor") + # 1. Create a KMS key for the user key = self._create_kms_key(name=_random_name("vm-root-key"), profile_id=self.default_profile.id, apiclient=self.user_apiclient)
diff --git a/test/integration/smoke/test_network_extension_namespace.py b/test/integration/smoke/test_network_extension_namespace.py index 5452cd4..f2484d7 100644 --- a/test/integration/smoke/test_network_extension_namespace.py +++ b/test/integration/smoke/test_network_extension_namespace.py
@@ -859,6 +859,8 @@ excluded from the check — the connectivity failure will surface naturally when the test later tries to deploy scripts. """ + if self.hv.lower() != 'kvm': + raise self.skipTest("Skipping test case for non-kvm hypervisor") if tools is None: tools = ['arping', 'dnsmasq', 'haproxy'] if not self.kvm_host_configs: @@ -2387,7 +2389,7 @@ @attr(tags=["advanced", "smoke"], required_hardware="true") def test_09_vpc_source_nat_ip_update(self): """Update VPC source NAT IP and verify old/new source NAT flags flip correctly.""" - self._check_kvm_host_prerequisites(['arping']) + self._check_kvm_host_prerequisites(['ip', 'arping', 'dnsmasq', 'haproxy']) svc = VPC_NETWORK_SERVICES _nw_offering, ext_name = self._setup_extension_nsp_offering(
diff --git a/test/integration/smoke/test_nonstrict_affinity_group.py b/test/integration/smoke/test_nonstrict_affinity_group.py index 1b9b0c0..3e9890d 100644 --- a/test/integration/smoke/test_nonstrict_affinity_group.py +++ b/test/integration/smoke/test_nonstrict_affinity_group.py
@@ -20,6 +20,7 @@ """ import logging +import time from marvin.codes import FAILED from nose.plugins.attrib import attr @@ -38,7 +39,6 @@ from marvin.lib.common import (get_domain, get_zone, - get_template, get_test_template) @@ -209,6 +209,9 @@ vm_2_host_id, msg="Both VMs of affinity group %s are on the same host" % self.affinity_group.name) + + time.sleep(10) + # 4. Migrate vm-2 to same host as vm-1 self.virtual_machine_2.migrate( self.apiclient, @@ -362,6 +365,8 @@ vm_12_host_id, msg="Both VMs of affinity group %s are on the different host" % self.affinity_group.name) + time.sleep(10) + # 4. Migrate vm-12 to different host as vm-11 self.virtual_machine_12.migrate( self.apiclient
diff --git a/test/integration/smoke/test_usage.py b/test/integration/smoke/test_usage.py index fef0d8f..5010709 100644 --- a/test/integration/smoke/test_usage.py +++ b/test/integration/smoke/test_usage.py
@@ -43,6 +43,7 @@ list_storage_pools, find_storage_pool_type) +import time class Services: @@ -782,6 +783,7 @@ ) rool_volume_pool = rool_volume_pool_response[0] try: + time.sleep(10) self.virtual_machine.attach_volume(self.apiclient,volume_uploaded) except Exception as e: self.debug("Exception %s: " % e) @@ -1744,16 +1746,9 @@ domainid=cls.account.domainid, serviceofferingid=cls.service_offering.id ) - cls.public_ip = PublicIPAddress.create( - cls.api_client, - accountid=cls.virtual_machine.account, - zoneid=cls.virtual_machine.zoneid, - domainid=cls.virtual_machine.domainid, - services=cls.services["server"] - ) src_nat_list = PublicIPAddress.list( cls.api_client, - accountid=cls.virtual_machine.account, + account=cls.virtual_machine.account, zoneid=cls.virtual_machine.zoneid, domainid=cls.virtual_machine.domainid, issourcenat=True
diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 8df0b99..2bb4eff 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py
@@ -858,6 +858,7 @@ zoneid=self.zone.id ) + time.sleep(10) small_virtual_machine.attach_volume(self.apiclient, vol1) self.debug("Destroy VM - ID: %s" % small_virtual_machine.id)
diff --git a/test/integration/smoke/test_vm_strict_host_tags.py b/test/integration/smoke/test_vm_strict_host_tags.py index aac3e1e..26d36d7 100644 --- a/test/integration/smoke/test_vm_strict_host_tags.py +++ b/test/integration/smoke/test_vm_strict_host_tags.py
@@ -22,6 +22,7 @@ from marvin.lib.common import (get_domain, get_zone) from nose.plugins.attrib import attr +import time class TestVMDeploymentPlannerStrictTags(cloudstackTestCase): @@ -529,6 +530,8 @@ self.cleanup.append(vm) self.assertEqual(self.host_h1.id, vm.hostid, "VM instance was not deployed on target host ID") Host.update(self.apiclient, id=self.host_h2.id, hosttags="h1,t1,v1") + + time.sleep(10) # Wait for VM to boot into OS before migrating the VM vm.migrate(self.apiclient, self.host_h2.id) migrated_vm = VirtualMachine.list(self.apiclient, id=vm.id, listall=True)[0] self.assertEqual(migrated_vm.hostid, self.host_h2.id, "VM was not migratd") @@ -545,6 +548,7 @@ self.assertEqual(self.host_h1.id, vm.hostid, "VM instance was not deployed on target host ID") Host.update(self.apiclient, id=self.host_h2.id, hosttags="h2,t2,v2") try: + time.sleep(10) # Wait for VM to boot into OS before migrating the VM vm.migrate(self.apiclient, self.host_h2.id) VirtualMachine.list(self.apiclient, id=vm.id, listall=True)[0] self.fail("VM should not be migrated")
diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index 0e46f2e..15e26f3 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java
@@ -128,9 +128,9 @@ javax.net.ssl.TrustManager tm = new TrustAllTrustManager(); trustAllCerts[0] = tm; javax.net.ssl.SSLContext sc = SSLUtils.getSSLContext(); + sc.init(null, trustAllCerts, null); javax.net.ssl.SSLSessionContext sslsc = sc.getServerSessionContext(); sslsc.setSessionTimeout(0); - sc.init(null, trustAllCerts, null); javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(new SecureSSLSocketFactory(sc)); }