| #!/usr/bin/env python |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| """ P1 tests for high availability |
| """ |
| #Import Local Modules |
| import marvin |
| from nose.plugins.attrib import attr |
| from marvin.cloudstackTestCase import * |
| from marvin.cloudstackAPI import * |
| from marvin.integration.lib.utils import * |
| from marvin.integration.lib.base import * |
| from marvin.integration.lib.common import * |
| from marvin import remoteSSHClient |
| import datetime |
| |
| |
| class Services: |
| """Test network offering Services |
| """ |
| |
| def __init__(self): |
| self.services = { |
| "account": { |
| "email": "test@test.com", |
| "firstname": "HA", |
| "lastname": "HA", |
| "username": "HA", |
| # Random characters are appended for unique |
| # username |
| "password": "password", |
| }, |
| "service_offering": { |
| "name": "Tiny Instance", |
| "displaytext": "Tiny Instance", |
| "cpunumber": 1, |
| "cpuspeed": 100, # in MHz |
| "memory": 128, # In MBs |
| }, |
| "lbrule": { |
| "name": "SSH", |
| "alg": "roundrobin", |
| # Algorithm used for load balancing |
| "privateport": 22, |
| "publicport": 2222, |
| }, |
| "natrule": { |
| "privateport": 22, |
| "publicport": 22, |
| "protocol": "TCP" |
| }, |
| "fw_rule": { |
| "startport": 1, |
| "endport": 6000, |
| "cidr": '55.55.0.0/11', |
| # Any network (For creating FW rule) |
| }, |
| "virtual_machine": { |
| "displayname": "VM", |
| "username": "root", |
| "password": "password", |
| "ssh_port": 22, |
| "hypervisor": 'XenServer', |
| # Hypervisor type should be same as |
| # hypervisor type of cluster |
| "privateport": 22, |
| "publicport": 22, |
| "protocol": 'TCP', |
| }, |
| "templates": { |
| "displaytext": "Public Template", |
| "name": "Public template", |
| "ostype": 'CentOS 5.3 (64-bit)', |
| "url": "http://download.cloud.com/releases/2.0.0/UbuntuServer-10-04-64bit.vhd.bz2", |
| "hypervisor": 'XenServer', |
| "format": 'VHD', |
| "isfeatured": True, |
| "ispublic": True, |
| "isextractable": True, |
| "templatefilter": 'self', |
| }, |
| "ostype": 'CentOS 5.3 (64-bit)', |
| # Cent OS 5.3 (64 bit) |
| "sleep": 60, |
| "timeout": 100, |
| "mode": 'advanced' |
| } |
| |
| |
| class TestHighAvailability(cloudstackTestCase): |
| |
| @classmethod |
| def setUpClass(cls): |
| |
| cls.api_client = super( |
| TestHighAvailability, |
| cls |
| ).getClsTestClient().getApiClient() |
| cls.services = Services().services |
| # Get Zone, Domain and templates |
| cls.domain = get_domain( |
| cls.api_client, |
| cls.services |
| ) |
| cls.zone = get_zone( |
| cls.api_client, |
| cls.services |
| ) |
| cls.pod = get_pod( |
| cls.api_client, |
| zoneid=cls.zone.id, |
| services=cls.services |
| ) |
| cls.template = get_template( |
| cls.api_client, |
| cls.zone.id, |
| cls.services["ostype"] |
| ) |
| cls.services["virtual_machine"]["zoneid"] = cls.zone.id |
| cls.services["virtual_machine"]["template"] = cls.template.id |
| |
| cls.service_offering = ServiceOffering.create( |
| cls.api_client, |
| cls.services["service_offering"], |
| offerha=True |
| ) |
| cls._cleanup = [ |
| cls.service_offering, |
| ] |
| return |
| |
| @classmethod |
| def tearDownClass(cls): |
| try: |
| #Cleanup resources used |
| cleanup_resources(cls.api_client, cls._cleanup) |
| except Exception as e: |
| raise Exception("Warning: Exception during cleanup : %s" % e) |
| return |
| |
| def setUp(self): |
| self.apiclient = self.testClient.getApiClient() |
| self.dbclient = self.testClient.getDbConnection() |
| self.account = Account.create( |
| self.apiclient, |
| self.services["account"], |
| admin=True, |
| domainid=self.domain.id |
| ) |
| self.cleanup = [self.account] |
| return |
| |
| def tearDown(self): |
| try: |
| #Clean up, terminate the created accounts, domains etc |
| cleanup_resources(self.apiclient, self.cleanup) |
| self.testClient.close() |
| except Exception as e: |
| raise Exception("Warning: Exception during cleanup : %s" % e) |
| return |
| |
| @attr(tags = ["advanced", "advancedns", "multihost"]) |
| def test_01_host_maintenance_mode(self): |
| """Test host maintenance mode |
| """ |
| |
| |
| # Validate the following |
| # 1. Create Vms. Acquire IP. Create port forwarding & load balancing |
| # rules for Vms. |
| # 2. Host 1: put to maintenance mode. All Vms should failover to Host |
| # 2 in cluster. Vms should be in running state. All port forwarding |
| # rules and load balancing Rules should work. |
| # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host |
| # 2 should succeed. |
| # 4. Host 1: cancel maintenance mode. |
| # 5. Host 2 : put to maintenance mode. All Vms should failover to |
| # Host 1 in cluster. |
| # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on |
| # host 1 should succeed. |
| |
| hosts = Host.list( |
| self.apiclient, |
| zoneid=self.zone.id, |
| resourcestate='Enabled', |
| type='Routing' |
| ) |
| self.assertEqual( |
| isinstance(hosts, list), |
| True, |
| "List hosts should return valid host response" |
| ) |
| self.assertGreaterEqual( |
| len(hosts), |
| 2, |
| "There must be two hosts present in a cluster" |
| ) |
| self.debug("Checking HA with hosts: %s, %s" % ( |
| hosts[0].name, |
| hosts[1].name |
| )) |
| self.debug("Deploying VM in account: %s" % self.account.name) |
| # Spawn an instance in that network |
| virtual_machine = VirtualMachine.create( |
| self.apiclient, |
| self.services["virtual_machine"], |
| accountid=self.account.name, |
| domainid=self.account.account.domainid, |
| serviceofferingid=self.service_offering.id |
| ) |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| self.debug("Deployed VM on host: %s" % vm.hostid) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in RUnning state" |
| ) |
| networks = Network.list( |
| self.apiclient, |
| account=self.account.name, |
| domainid=self.account.account.domainid, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(networks, list), |
| True, |
| "List networks should return valid list for the account" |
| ) |
| network = networks[0] |
| |
| self.debug("Associating public IP for account: %s" % |
| self.account.name) |
| public_ip = PublicIPAddress.create( |
| self.apiclient, |
| accountid=self.account.name, |
| zoneid=self.zone.id, |
| domainid=self.account.account.domainid, |
| networkid=network.id |
| ) |
| |
| self.debug("Associated %s with network %s" % ( |
| public_ip.ipaddress.ipaddress, |
| network.id |
| )) |
| self.debug("Creating PF rule for IP address: %s" % |
| public_ip.ipaddress.ipaddress) |
| nat_rule = NATRule.create( |
| self.apiclient, |
| virtual_machine, |
| self.services["natrule"], |
| ipaddressid=public_ip.ipaddress.id |
| ) |
| |
| self.debug("Creating LB rule on IP with NAT: %s" % |
| public_ip.ipaddress.ipaddress) |
| |
| # Create Load Balancer rule on IP already having NAT rule |
| lb_rule = LoadBalancerRule.create( |
| self.apiclient, |
| self.services["lbrule"], |
| ipaddressid=public_ip.ipaddress.id, |
| accountid=self.account.name |
| ) |
| self.debug("Created LB rule with ID: %s" % lb_rule.id) |
| |
| # Should be able to SSH VM |
| try: |
| self.debug("SSH into VM: %s" % virtual_machine.id) |
| ssh = virtual_machine.get_ssh_client( |
| ipaddress=public_ip.ipaddress.ipaddress) |
| except Exception as e: |
| self.fail("SSH Access failed for %s: %s" % \ |
| (virtual_machine.ipaddress, e) |
| ) |
| |
| first_host = vm.hostid |
| self.debug("Enabling maintenance mode for host %s" % vm.hostid) |
| cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() |
| cmd.id = first_host |
| self.apiclient.prepareHostForMaintenance(cmd) |
| |
| self.debug("Waiting for SSVMs to come up") |
| wait_for_ssvms( |
| self.apiclient, |
| zoneid=self.zone.id, |
| podid=self.pod.id, |
| ) |
| |
| timeout = self.services["timeout"] |
| # Poll and check state of VM while it migrates from one host to another |
| while True: |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| |
| self.debug("VM 1 state: %s" % vm.state) |
| if vm.state in ["Stopping", |
| "Stopped", |
| "Running", |
| "Starting", |
| "Migrating"]: |
| if vm.state == "Running": |
| break |
| else: |
| time.sleep(self.services["sleep"]) |
| timeout = timeout - 1 |
| else: |
| self.fail( |
| "VM migration from one-host-to-other failed while enabling maintenance" |
| ) |
| second_host = vm.hostid |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "VM should be in Running state after enabling host maintenance" |
| ) |
| # Should be able to SSH VM |
| try: |
| self.debug("SSH into VM: %s" % virtual_machine.id) |
| ssh = virtual_machine.get_ssh_client( |
| ipaddress=public_ip.ipaddress.ipaddress) |
| except Exception as e: |
| self.fail("SSH Access failed for %s: %s" % \ |
| (virtual_machine.ipaddress, e) |
| ) |
| self.debug("Deploying VM in account: %s" % self.account.name) |
| # Spawn an instance on other host |
| virtual_machine_2 = VirtualMachine.create( |
| self.apiclient, |
| self.services["virtual_machine"], |
| accountid=self.account.name, |
| domainid=self.account.account.domainid, |
| serviceofferingid=self.service_offering.id |
| ) |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine_2.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| self.debug("Deployed VM on host: %s" % vm.hostid) |
| self.debug("VM 2 state: %s" % vm.state) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in Running state" |
| ) |
| |
| self.debug("Canceling host maintenance for ID: %s" % first_host) |
| cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() |
| cmd.id = first_host |
| self.apiclient.cancelHostMaintenance(cmd) |
| self.debug("Maintenance mode canceled for host: %s" % first_host) |
| |
| self.debug("Enabling maintenance mode for host %s" % second_host) |
| cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() |
| cmd.id = second_host |
| self.apiclient.prepareHostForMaintenance(cmd) |
| self.debug("Maintenance mode enabled for host: %s" % second_host) |
| |
| self.debug("Waiting for SSVMs to come up") |
| wait_for_ssvms( |
| self.apiclient, |
| zoneid=self.zone.id, |
| podid=self.pod.id, |
| ) |
| |
| # Poll and check the status of VMs |
| timeout = self.services["timeout"] |
| while True: |
| vms = VirtualMachine.list( |
| self.apiclient, |
| account=self.account.name, |
| domainid=self.account.account.domainid, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| self.debug( |
| "VM state after enabling maintenance on first host: %s" % |
| vm.state) |
| if vm.state in [ |
| "Stopping", |
| "Stopped", |
| "Running", |
| "Starting", |
| "Migrating" |
| ]: |
| if vm.state == "Running": |
| break |
| else: |
| time.sleep(self.services["sleep"]) |
| timeout = timeout - 1 |
| else: |
| self.fail( |
| "VM migration from one-host-to-other failed while enabling maintenance" |
| ) |
| |
| # Poll and check the status of VMs |
| timeout = self.services["timeout"] |
| while True: |
| vms = VirtualMachine.list( |
| self.apiclient, |
| account=self.account.name, |
| domainid=self.account.account.domainid, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[1] |
| self.debug( |
| "VM state after enabling maintenance on first host: %s" % |
| vm.state) |
| if vm.state in [ |
| "Stopping", |
| "Stopped", |
| "Running", |
| "Starting", |
| "Migrating" |
| ]: |
| if vm.state == "Running": |
| break |
| else: |
| time.sleep(self.services["sleep"]) |
| timeout = timeout - 1 |
| else: |
| self.fail( |
| "VM migration from one-host-to-other failed while enabling maintenance" |
| ) |
| |
| for vm in vms: |
| self.debug( |
| "VM states after enabling maintenance mode on host: %s - %s" % |
| (first_host, vm.state)) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in Running state" |
| ) |
| |
| # Spawn an instance on other host |
| virtual_machine_3 = VirtualMachine.create( |
| self.apiclient, |
| self.services["virtual_machine"], |
| accountid=self.account.name, |
| domainid=self.account.account.domainid, |
| serviceofferingid=self.service_offering.id |
| ) |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine_3.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| |
| self.debug("Deployed VM on host: %s" % vm.hostid) |
| self.debug("VM 3 state: %s" % vm.state) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in Running state" |
| ) |
| |
| # Should be able to SSH VM |
| try: |
| self.debug("SSH into VM: %s" % virtual_machine.id) |
| ssh = virtual_machine.get_ssh_client( |
| ipaddress=public_ip.ipaddress.ipaddress) |
| except Exception as e: |
| self.fail("SSH Access failed for %s: %s" % \ |
| (virtual_machine.ipaddress, e) |
| ) |
| |
| self.debug("Canceling host maintenance for ID: %s" % second_host) |
| cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() |
| cmd.id = second_host |
| self.apiclient.cancelHostMaintenance(cmd) |
| self.debug("Maintenance mode canceled for host: %s" % second_host) |
| |
| self.debug("Waiting for SSVMs to come up") |
| wait_for_ssvms( |
| self.apiclient, |
| zoneid=self.zone.id, |
| podid=self.pod.id, |
| ) |
| return |
| |
| @attr(tags = ["advanced", "advancedns", "multihost"]) |
| def test_02_host_maintenance_mode_with_activities(self): |
| """Test host maintenance mode with activities |
| """ |
| |
| |
| # Validate the following |
| # 1. Create Vms. Acquire IP. Create port forwarding & load balancing |
| # rules for Vms. |
| # 2. While activities are ongoing: Create snapshots, recurring |
| # snapshots, create templates, download volumes, Host 1: put to |
| # maintenance mode. All Vms should failover to Host 2 in cluster |
| # Vms should be in running state. All port forwarding rules and |
| # load balancing Rules should work. |
| # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host |
| # 2 should succeed. All ongoing activities in step 3 should succeed |
| # 4. Host 1: cancel maintenance mode. |
| # 5. While activities are ongoing: Create snapshots, recurring |
| # snapshots, create templates, download volumes, Host 2: put to |
| # maintenance mode. All Vms should failover to Host 1 in cluster. |
| # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on |
| # host 1 should succeed. All ongoing activities in step 6 should |
| # succeed. |
| |
| hosts = Host.list( |
| self.apiclient, |
| zoneid=self.zone.id, |
| resourcestate='Enabled', |
| type='Routing' |
| ) |
| self.assertEqual( |
| isinstance(hosts, list), |
| True, |
| "List hosts should return valid host response" |
| ) |
| self.assertGreaterEqual( |
| len(hosts), |
| 2, |
| "There must be two hosts present in a cluster" |
| ) |
| self.debug("Checking HA with hosts: %s, %s" % ( |
| hosts[0].name, |
| hosts[1].name |
| )) |
| self.debug("Deploying VM in account: %s" % self.account.name) |
| # Spawn an instance in that network |
| virtual_machine = VirtualMachine.create( |
| self.apiclient, |
| self.services["virtual_machine"], |
| accountid=self.account.name, |
| domainid=self.account.account.domainid, |
| serviceofferingid=self.service_offering.id |
| ) |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| self.debug("Deployed VM on host: %s" % vm.hostid) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in RUnning state" |
| ) |
| networks = Network.list( |
| self.apiclient, |
| account=self.account.name, |
| domainid=self.account.account.domainid, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(networks, list), |
| True, |
| "List networks should return valid list for the account" |
| ) |
| network = networks[0] |
| |
| self.debug("Associating public IP for account: %s" % |
| self.account.name) |
| public_ip = PublicIPAddress.create( |
| self.apiclient, |
| accountid=self.account.name, |
| zoneid=self.zone.id, |
| domainid=self.account.account.domainid, |
| networkid=network.id |
| ) |
| |
| self.debug("Associated %s with network %s" % ( |
| public_ip.ipaddress.ipaddress, |
| network.id |
| )) |
| self.debug("Creating PF rule for IP address: %s" % |
| public_ip.ipaddress.ipaddress) |
| nat_rule = NATRule.create( |
| self.apiclient, |
| virtual_machine, |
| self.services["natrule"], |
| ipaddressid=public_ip.ipaddress.id |
| ) |
| |
| self.debug("Creating LB rule on IP with NAT: %s" % |
| public_ip.ipaddress.ipaddress) |
| |
| # Create Load Balancer rule on IP already having NAT rule |
| lb_rule = LoadBalancerRule.create( |
| self.apiclient, |
| self.services["lbrule"], |
| ipaddressid=public_ip.ipaddress.id, |
| accountid=self.account.name |
| ) |
| self.debug("Created LB rule with ID: %s" % lb_rule.id) |
| |
| # Should be able to SSH VM |
| try: |
| self.debug("SSH into VM: %s" % virtual_machine.id) |
| ssh = virtual_machine.get_ssh_client( |
| ipaddress=public_ip.ipaddress.ipaddress) |
| except Exception as e: |
| self.fail("SSH Access failed for %s: %s" % \ |
| (virtual_machine.ipaddress, e) |
| ) |
| # Get the Root disk of VM |
| volumes = list_volumes( |
| self.apiclient, |
| virtualmachineid=virtual_machine.id, |
| type='ROOT', |
| listall=True |
| ) |
| volume = volumes[0] |
| self.debug( |
| "Root volume of VM(%s): %s" % ( |
| virtual_machine.name, |
| volume.name |
| )) |
| # Create a snapshot from the ROOTDISK |
| self.debug("Creating snapshot on ROOT volume: %s" % volume.name) |
| snapshot = Snapshot.create(self.apiclient, volumes[0].id) |
| self.debug("Snapshot created: ID - %s" % snapshot.id) |
| |
| snapshots = list_snapshots( |
| self.apiclient, |
| id=snapshot.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(snapshots, list), |
| True, |
| "Check list response returns a valid list" |
| ) |
| self.assertNotEqual( |
| snapshots, |
| None, |
| "Check if result exists in list snapshots call" |
| ) |
| self.assertEqual( |
| snapshots[0].id, |
| snapshot.id, |
| "Check snapshot id in list resources call" |
| ) |
| |
| # Generate template from the snapshot |
| self.debug("Generating template from snapshot: %s" % snapshot.name) |
| template = Template.create_from_snapshot( |
| self.apiclient, |
| snapshot, |
| self.services["templates"] |
| ) |
| self.debug("Created template from snapshot: %s" % template.id) |
| |
| templates = list_templates( |
| self.apiclient, |
| templatefilter=\ |
| self.services["templates"]["templatefilter"], |
| id=template.id |
| ) |
| |
| self.assertEqual( |
| isinstance(templates, list), |
| True, |
| "List template call should return the newly created template" |
| ) |
| |
| self.assertEqual( |
| templates[0].isready, |
| True, |
| "The newly created template should be in ready state" |
| ) |
| |
| first_host = vm.hostid |
| self.debug("Enabling maintenance mode for host %s" % vm.hostid) |
| cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() |
| cmd.id = first_host |
| self.apiclient.prepareHostForMaintenance(cmd) |
| |
| self.debug("Waiting for SSVMs to come up") |
| wait_for_ssvms( |
| self.apiclient, |
| zoneid=self.zone.id, |
| podid=self.pod.id, |
| ) |
| |
| timeout = self.services["timeout"] |
| # Poll and check state of VM while it migrates from one host to another |
| while True: |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| |
| self.debug("VM 1 state: %s" % vm.state) |
| if vm.state in ["Stopping", |
| "Stopped", |
| "Running", |
| "Starting", |
| "Migrating"]: |
| if vm.state == "Running": |
| break |
| else: |
| time.sleep(self.services["sleep"]) |
| timeout = timeout - 1 |
| else: |
| self.fail( |
| "VM migration from one-host-to-other failed while enabling maintenance" |
| ) |
| second_host = vm.hostid |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "VM should be in Running state after enabling host maintenance" |
| ) |
| # Should be able to SSH VM |
| try: |
| self.debug("SSH into VM: %s" % virtual_machine.id) |
| ssh = virtual_machine.get_ssh_client( |
| ipaddress=public_ip.ipaddress.ipaddress) |
| except Exception as e: |
| self.fail("SSH Access failed for %s: %s" % \ |
| (virtual_machine.ipaddress, e) |
| ) |
| self.debug("Deploying VM in account: %s" % self.account.name) |
| # Spawn an instance on other host |
| virtual_machine_2 = VirtualMachine.create( |
| self.apiclient, |
| self.services["virtual_machine"], |
| accountid=self.account.name, |
| domainid=self.account.account.domainid, |
| serviceofferingid=self.service_offering.id |
| ) |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine_2.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| self.debug("Deployed VM on host: %s" % vm.hostid) |
| self.debug("VM 2 state: %s" % vm.state) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in Running state" |
| ) |
| |
| self.debug("Canceling host maintenance for ID: %s" % first_host) |
| cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() |
| cmd.id = first_host |
| self.apiclient.cancelHostMaintenance(cmd) |
| self.debug("Maintenance mode canceled for host: %s" % first_host) |
| |
| # Get the Root disk of VM |
| volumes = list_volumes( |
| self.apiclient, |
| virtualmachineid=virtual_machine_2.id, |
| type='ROOT', |
| listall=True |
| ) |
| volume = volumes[0] |
| self.debug( |
| "Root volume of VM(%s): %s" % ( |
| virtual_machine_2.name, |
| volume.name |
| )) |
| # Create a snapshot from the ROOTDISK |
| self.debug("Creating snapshot on ROOT volume: %s" % volume.name) |
| snapshot = Snapshot.create(self.apiclient, volumes[0].id) |
| self.debug("Snapshot created: ID - %s" % snapshot.id) |
| |
| snapshots = list_snapshots( |
| self.apiclient, |
| id=snapshot.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(snapshots, list), |
| True, |
| "Check list response returns a valid list" |
| ) |
| self.assertNotEqual( |
| snapshots, |
| None, |
| "Check if result exists in list snapshots call" |
| ) |
| self.assertEqual( |
| snapshots[0].id, |
| snapshot.id, |
| "Check snapshot id in list resources call" |
| ) |
| |
| # Generate template from the snapshot |
| self.debug("Generating template from snapshot: %s" % snapshot.name) |
| template = Template.create_from_snapshot( |
| self.apiclient, |
| snapshot, |
| self.services["templates"] |
| ) |
| self.debug("Created template from snapshot: %s" % template.id) |
| |
| templates = list_templates( |
| self.apiclient, |
| templatefilter=\ |
| self.services["templates"]["templatefilter"], |
| id=template.id |
| ) |
| |
| self.assertEqual( |
| isinstance(templates, list), |
| True, |
| "List template call should return the newly created template" |
| ) |
| |
| self.assertEqual( |
| templates[0].isready, |
| True, |
| "The newly created template should be in ready state" |
| ) |
| |
| self.debug("Enabling maintenance mode for host %s" % second_host) |
| cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() |
| cmd.id = second_host |
| self.apiclient.prepareHostForMaintenance(cmd) |
| self.debug("Maintenance mode enabled for host: %s" % second_host) |
| |
| self.debug("Waiting for SSVMs to come up") |
| wait_for_ssvms( |
| self.apiclient, |
| zoneid=self.zone.id, |
| podid=self.pod.id, |
| ) |
| |
| # Poll and check the status of VMs |
| timeout = self.services["timeout"] |
| while True: |
| vms = VirtualMachine.list( |
| self.apiclient, |
| account=self.account.name, |
| domainid=self.account.account.domainid, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| self.debug( |
| "VM state after enabling maintenance on first host: %s" % |
| vm.state) |
| if vm.state in ["Stopping", |
| "Stopped", |
| "Running", |
| "Starting", |
| "Migrating"]: |
| if vm.state == "Running": |
| break |
| else: |
| time.sleep(self.services["sleep"]) |
| timeout = timeout - 1 |
| else: |
| self.fail( |
| "VM migration from one-host-to-other failed while enabling maintenance" |
| ) |
| |
| # Poll and check the status of VMs |
| timeout = self.services["timeout"] |
| while True: |
| vms = VirtualMachine.list( |
| self.apiclient, |
| account=self.account.name, |
| domainid=self.account.account.domainid, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[1] |
| self.debug( |
| "VM state after enabling maintenance on first host: %s" % |
| vm.state) |
| if vm.state in ["Stopping", |
| "Stopped", |
| "Running", |
| "Starting", |
| "Migrating"]: |
| if vm.state == "Running": |
| break |
| else: |
| time.sleep(self.services["sleep"]) |
| timeout = timeout - 1 |
| else: |
| self.fail( |
| "VM migration from one-host-to-other failed while enabling maintenance" |
| ) |
| |
| for vm in vms: |
| self.debug( |
| "VM states after enabling maintenance mode on host: %s - %s" % |
| (first_host, vm.state)) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in Running state" |
| ) |
| |
| # Spawn an instance on other host |
| virtual_machine_3 = VirtualMachine.create( |
| self.apiclient, |
| self.services["virtual_machine"], |
| accountid=self.account.name, |
| domainid=self.account.account.domainid, |
| serviceofferingid=self.service_offering.id |
| ) |
| vms = VirtualMachine.list( |
| self.apiclient, |
| id=virtual_machine_3.id, |
| listall=True |
| ) |
| self.assertEqual( |
| isinstance(vms, list), |
| True, |
| "List VMs should return valid response for deployed VM" |
| ) |
| self.assertNotEqual( |
| len(vms), |
| 0, |
| "List VMs should return valid response for deployed VM" |
| ) |
| vm = vms[0] |
| |
| self.debug("Deployed VM on host: %s" % vm.hostid) |
| self.debug("VM 3 state: %s" % vm.state) |
| self.assertEqual( |
| vm.state, |
| "Running", |
| "Deployed VM should be in Running state" |
| ) |
| |
| self.debug("Canceling host maintenance for ID: %s" % second_host) |
| cmd = cancelHostMaintenance.cancelHostMaintenanceCmd() |
| cmd.id = second_host |
| self.apiclient.cancelHostMaintenance(cmd) |
| self.debug("Maintenance mode canceled for host: %s" % second_host) |
| |
| self.debug("Waiting for SSVMs to come up") |
| wait_for_ssvms( |
| self.apiclient, |
| zoneid=self.zone.id, |
| podid=self.pod.id, |
| ) |
| return |
