CLOUDSTACK-8915 - Making sure cleanup resources passes
- Due to an issue with VPC routers (CLOUDSTACK-8935) we are not able to destroy networks
before destroying the routers
- Added a forcestop/destroy routers inside the tearDown to make sure it passes. The issue
will be addressed in a separate PR
- Make sure the routers list is cleaned after destroy_routers() is called
- Populate routers list after the router is recreated
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/7ed81a69
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/7ed81a69
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/7ed81a69
Branch: refs/heads/master
Commit: 7ed81a6990956ef577c8e193fadb195546ed3181
Parents: b1fb153
Author: Wilder Rodrigues <wrodrigues@schubergphilis.com>
Authored: Mon Oct 5 13:26:55 2015 +0200
Committer: Wilder Rodrigues <wrodrigues@schubergphilis.com>
Committed: Wed Oct 7 11:14:23 2015 +0200
----------------------------------------------------------------------
.../test_routers_iptables_default_policy.py | 73 +++++++++++++------
.../component/test_routers_network_ops.py | 19 ++---
.../integration/component/test_vpc_redundant.py | 63 ++++++++++++----
.../component/test_vpc_router_nics.py | 75 ++++++++++++--------
4 files changed, 154 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7ed81a69/test/integration/component/test_routers_iptables_default_policy.py
----------------------------------------------------------------------
diff --git a/test/integration/component/test_routers_iptables_default_policy.py b/test/integration/component/test_routers_iptables_default_policy.py
index b72e45f..a1bec4c 100644
--- a/test/integration/component/test_routers_iptables_default_policy.py
+++ b/test/integration/component/test_routers_iptables_default_policy.py
@@ -230,21 +230,19 @@ class TestVPCIpTablesPolicies(cloudstackTestCase):
admin=True,
domainid=cls.domain.id)
- cls._cleanup = [cls.account]
-
cls.service_offering = ServiceOffering.create(
cls.apiclient,
cls.services["service_offering"])
- cls._cleanup.append(cls.service_offering)
cls.logger = logging.getLogger('TestVPCIpTablesPolicies')
cls.stream_handler = logging.StreamHandler()
cls.logger.setLevel(logging.DEBUG)
cls.logger.addHandler(cls.stream_handler)
- cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering,
cls.account, cls.zone, cls._cleanup, cls.logger)
+ cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering,
cls.account, cls.zone, cls.logger)
+ cls._cleanup = [cls.service_offering, cls.account]
return
@classmethod
@@ -274,6 +272,16 @@ class TestVPCIpTablesPolicies(cloudstackTestCase):
account=self.account.name,
domainid=self.account.domainid)
+ self.cleanup = [self.vpc, self.vpc_off]
+ self.entity_manager.set_cleanup(self.cleanup)
+ return
+
+ def tearDown(self):
+ try:
+ self.entity_manager.destroy_routers()
+ cleanup_resources(self.apiclient, self.cleanup)
+ except Exception as e:
+ raise Exception("Warning: Exception during cleanup : %s" % e)
return
@attr(tags=["advanced", "intervlan"], required_hardware="true")
@@ -361,21 +369,18 @@ class TestRouterIpTablesPolicies(cloudstackTestCase):
admin=True,
domainid=cls.domain.id)
- cls._cleanup = [cls.account]
-
cls.service_offering = ServiceOffering.create(
cls.apiclient,
cls.services["service_offering"])
- cls._cleanup.append(cls.service_offering)
-
cls.logger = logging.getLogger('TestRouterIpTablesPolicies')
cls.stream_handler = logging.StreamHandler()
cls.logger.setLevel(logging.DEBUG)
cls.logger.addHandler(cls.stream_handler)
- cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering,
cls.account, cls.zone, cls._cleanup, cls.logger)
+ cls.entity_manager = EntityManager(cls.apiclient, cls.services, cls.service_offering,
cls.account, cls.zone, cls.logger)
+ cls._cleanup = [cls.service_offering, cls.account]
return
@classmethod
@@ -386,6 +391,18 @@ class TestRouterIpTablesPolicies(cloudstackTestCase):
raise Exception("Warning: Exception during cleanup : %s" % e)
return
+ def setUp(self):
+ self.cleanup = []
+ self.entity_manager.set_cleanup(self.cleanup)
+ return
+
+ def tearDown(self):
+ try:
+ cleanup_resources(self.apiclient, self.cleanup)
+ except Exception as e:
+ raise Exception("Warning: Exception during cleanup : %s" % e)
+ return
+
@attr(tags=["advanced", "intervlan"], required_hardware="true")
def test_02_routervm_iptables_policies(self):
""" Test iptables default INPUT/FORWARD policy on RouterVM """
@@ -442,18 +459,21 @@ class TestRouterIpTablesPolicies(cloudstackTestCase):
class EntityManager(object):
- def __init__(self, apiclient, services, service_offering, account, zone, cleanup, logger):
+ def __init__(self, apiclient, services, service_offering, account, zone, logger):
self.apiclient = apiclient
self.services = services
self.service_offering = service_offering
self.account = account
self.zone = zone
- self.cleanup = cleanup
self.logger = logger
+ self.cleanup = []
self.networks = []
self.routers = []
self.ips = []
+
+ def set_cleanup(self, cleanup):
+ self.cleanup = cleanup
def add_nat_rules(self, vpc_id):
for o in self.networks:
@@ -514,7 +534,6 @@ class EntityManager(object):
conservemode=False)
nw_off.update(self.apiclient, state='Enabled')
- self.cleanup.append(nw_off)
self.logger.debug('Created and Enabled NetworkOffering')
self.services["network"]["name"] = "NETWORK-" + str(gateway)
@@ -528,13 +547,18 @@ class EntityManager(object):
zoneid=self.zone.id,
gateway=gateway,
vpcid=vpc_id)
+
self.logger.debug("Created network with ID: %s" % obj_network.id)
except Exception, e:
raise Exception('Unable to create a Network with offering=%s because of %s '
% (net_offerring, e))
o = networkO(obj_network)
- o.add_vm(self.deployvm_in_network(obj_network))
+ vm1 = self.deployvm_in_network(obj_network)
+ self.cleanup.insert(1, obj_network)
+ self.cleanup.insert(2, nw_off)
+
+ o.add_vm(vm1)
self.networks.append(o)
return o
@@ -548,7 +572,9 @@ class EntityManager(object):
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id,
networkids=[str(network.id)])
+
self.logger.debug('Created VM=%s in network=%s' % (vm.id, network.name))
+ self.cleanup.insert(0, vm)
return vm
except:
raise Exception('Unable to create VM in a Network=%s' % network.name)
@@ -562,6 +588,8 @@ class EntityManager(object):
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
+
+ self.cleanup.insert(0, vm)
self.logger.debug('Created VM=%s' % vm.id)
return vm
except:
@@ -590,22 +618,23 @@ class EntityManager(object):
return self.routers
- def stop_router(self):
+ def stop_router(self, router):
self.logger.debug('Stopping router')
- for router in self.routers:
- cmd = stopRouter.stopRouterCmd()
- cmd.id = router.id
- self.apiclient.stopRouter(cmd)
+ cmd = stopRouter.stopRouterCmd()
+ cmd.id = router.id
+ self.apiclient.stopRouter(cmd)
- def destroy_router(self):
- self.logger.debug('Destroying router')
+ def destroy_routers(self):
+ self.logger.debug('Destroying routers')
for router in self.routers:
+ self.stop_router(router)
cmd = destroyRouter.destroyRouterCmd()
cmd.id = router.id
self.apiclient.destroyRouter(cmd)
+ self.routers = []
- def start_router(self):
- self.logger.debug('Starting router')
+ def start_routers(self):
+ self.logger.debug('Starting routers')
for router in self.routers:
cmd = startRouter.startRouterCmd()
cmd.id = router.id
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7ed81a69/test/integration/component/test_routers_network_ops.py
----------------------------------------------------------------------
diff --git a/test/integration/component/test_routers_network_ops.py b/test/integration/component/test_routers_network_ops.py
index 9937a4e..7f1e4aa 100644
--- a/test/integration/component/test_routers_network_ops.py
+++ b/test/integration/component/test_routers_network_ops.py
@@ -110,6 +110,11 @@ class TestCreatePFOnStoppedRouter(cloudstackTestCase):
raise Exception("Warning: Exception during cleanup : %s" % e)
return
+ def setUp(self):
+ self.apiclient = self.testClient.getApiClient()
+ self.cleanup = []
+ return
+
def tearDown(self):
try:
# Clean up, terminate the created resources
@@ -118,10 +123,6 @@ class TestCreatePFOnStoppedRouter(cloudstackTestCase):
raise Exception("Warning: Exception during cleanup : %s" % e)
return
- def setUp(self):
- self.apiclient = self.testClient.getApiClient()
- self.cleanup = []
- return
@attr(tags=["advanced", "advancedns"], required_hardware="true")
def test_01_CreatePFOnStoppedRouter(self):
@@ -322,6 +323,11 @@ class TestCreateLBOnStoppedRouter(cloudstackTestCase):
raise Exception("Warning: Exception during cleanup : %s" % e)
return
+ def setUp(self):
+ self.apiclient = self.testClient.getApiClient()
+ self.cleanup = []
+ return
+
def tearDown(self):
try:
cleanup_resources(self.apiclient, self.cleanup)
@@ -329,11 +335,6 @@ class TestCreateLBOnStoppedRouter(cloudstackTestCase):
raise Exception("Warning: Exception during cleanup : %s" % e)
return
- def setUp(self):
- self.apiclient = self.testClient.getApiClient()
- self.cleanup = []
- return
-
@attr(tags=["advanced", "advancedns"], required_hardware="true")
def test_01_CreateLBOnStoppedRouter(self):
"""Stop existing Router, add LB rule and check we can reach the VM"""
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7ed81a69/test/integration/component/test_vpc_redundant.py
----------------------------------------------------------------------
diff --git a/test/integration/component/test_vpc_redundant.py b/test/integration/component/test_vpc_redundant.py
index e1a6ecb..11fabf6 100644
--- a/test/integration/component/test_vpc_redundant.py
+++ b/test/integration/component/test_vpc_redundant.py
@@ -22,6 +22,7 @@ from nose.plugins.attrib import attr
from marvin.cloudstackTestCase import cloudstackTestCase
from marvin.lib.base import (stopRouter,
startRouter,
+ destroyRouter,
Account,
VpcOffering,
VPC,
@@ -242,7 +243,6 @@ class TestVPCRedundancy(cloudstackTestCase):
admin=True,
domainid=self.domain.id)
- self._cleanup = [self.account]
self.logger.debug("Creating a VPC offering..")
self.vpc_off = VpcOffering.create(
self.apiclient,
@@ -260,6 +260,17 @@ class TestVPCRedundancy(cloudstackTestCase):
zoneid=self.zone.id,
account=self.account.name,
domainid=self.account.domainid)
+
+ self.cleanup = [self.vpc, self.vpc_off, self.account]
+ return
+
+ def tearDown(self):
+ try:
+ #Stop/Destroy the routers so we are able to remove the networks. Issue CLOUDSTACK-8935
+ self.destroy_routers()
+ cleanup_resources(self.api_client, self.cleanup)
+ except Exception as e:
+ raise Exception("Warning: Exception during cleanup : %s" % e)
return
def query_routers(self, count=2, showall=False):
@@ -287,16 +298,29 @@ class TestVPCRedundancy(cloudstackTestCase):
if cnts[vals.index('MASTER')] != 1:
self.fail("No Master or too many master routers found %s" % cnts[vals.index('MASTER')])
- def stop_router(self, type):
+ def stop_router(self, router):
+ self.logger.debug('Stopping router %s' % router.id)
+ cmd = stopRouter.stopRouterCmd()
+ cmd.id = router.id
+ self.apiclient.stopRouter(cmd)
+
+ def stop_router_by_type(self, type):
self.check_master_status(2)
self.logger.debug('Stopping %s router' % type)
for router in self.routers:
if router.redundantstate == type:
- cmd = stopRouter.stopRouterCmd()
- cmd.id = router.id
- self.apiclient.stopRouter(cmd)
+ self.stop_router(router)
- def start_router(self):
+ def destroy_routers(self):
+ self.logger.debug('Destroying routers')
+ for router in self.routers:
+ self.stop_router(router)
+ cmd = destroyRouter.destroyRouterCmd()
+ cmd.id = router.id
+ self.apiclient.destroyRouter(cmd)
+ self.routers = []
+
+ def start_routers(self):
self.check_master_status(2, showall=True)
self.logger.debug('Starting stopped routers')
for router in self.routers:
@@ -317,6 +341,7 @@ class TestVPCRedundancy(cloudstackTestCase):
conservemode=False)
nw_off.update(self.apiclient, state='Enabled')
+
self.logger.debug('Created and Enabled NetworkOffering')
self.services["network"]["name"] = "NETWORK-" + str(gateway)
@@ -336,8 +361,14 @@ class TestVPCRedundancy(cloudstackTestCase):
except Exception, e:
self.fail('Unable to create a Network with offering=%s because of %s ' % (net_offerring,
e))
o = networkO(obj_network)
- o.add_vm(self.deployvm_in_network(obj_network))
- o.add_vm(self.deployvm_in_network(obj_network))
+
+ vm1 = self.deployvm_in_network(obj_network)
+ vm2 = self.deployvm_in_network(obj_network)
+ self.cleanup.insert(2, obj_network)
+ self.cleanup.insert(3, nw_off)
+
+ o.add_vm(vm1)
+ o.add_vm(vm2)
return o
def deployvm_in_network(self, network, host_id=None):
@@ -352,7 +383,9 @@ class TestVPCRedundancy(cloudstackTestCase):
networkids=[str(network.id)],
hostid=host_id
)
+
self.logger.debug('Created VM=%s in network=%s' % (vm.id, network.name))
+ self.cleanup.insert(0, vm)
return vm
except:
self.fail('Unable to create VM in a Network=%s' % network.name)
@@ -394,9 +427,10 @@ class TestVPCRedundancy(cloudstackTestCase):
traffictype='Ingress'
)
self.logger.debug('nwacl_nat=%s' % nwacl_nat.__dict__)
+
return nat_rule
- def check_ssh_into_vm(self, vm, public_ip, expectFail=False, retries=20):
+ def check_ssh_into_vm(self, vm, public_ip, expectFail=False, retries=5):
self.logger.debug("Checking if we can SSH into VM=%s on public_ip=%s (%r)" %
(vm.name, public_ip.ipaddress.ipaddress, expectFail))
vm.ssh_client = None
@@ -429,7 +463,7 @@ class TestVPCRedundancy(cloudstackTestCase):
self.add_nat_rules()
self.do_vpc_test(False)
- self.stop_router("MASTER")
+ self.stop_router_by_type("MASTER")
# wait for the backup router to transit to master state
time.sleep(30)
self.check_master_status(1)
@@ -440,7 +474,7 @@ class TestVPCRedundancy(cloudstackTestCase):
self.check_master_status(1)
self.do_vpc_test(True)
- self.start_router()
+ self.start_routers()
self.add_nat_rules()
time.sleep(45)
self.check_master_status(2)
@@ -456,8 +490,7 @@ class TestVPCRedundancy(cloudstackTestCase):
time.sleep(30)
self.check_master_status(2)
self.add_nat_rules()
- self.test_default_routes()
-
+ self.do_default_routes_test()
def delete_nat_rules(self):
for o in self.networks:
@@ -483,7 +516,7 @@ class TestVPCRedundancy(cloudstackTestCase):
for vm in o.get_vms():
self.check_ssh_into_vm(vm.get_vm(), vm.get_ip(), expectFail=expectFail, retries=retries)
- def test_default_routes(self):
+ def do_default_routes_test(self):
for o in self.networks:
for vmObj in o.get_vms():
ssh_command = "ping -c 3 8.8.8.8"
@@ -500,7 +533,7 @@ class TestVPCRedundancy(cloudstackTestCase):
self.logger.debug("Ping to google.com from VM")
result = str(ssh.execute(ssh_command))
- self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("0%
packet loss")))
+ self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("3
packets received")))
except Exception as e:
self.fail("SSH Access failed for %s: %s" % \
(vmObj.get_ip(), e)
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7ed81a69/test/integration/component/test_vpc_router_nics.py
----------------------------------------------------------------------
diff --git a/test/integration/component/test_vpc_router_nics.py b/test/integration/component/test_vpc_router_nics.py
index db4e27a..cb0734d 100644
--- a/test/integration/component/test_vpc_router_nics.py
+++ b/test/integration/component/test_vpc_router_nics.py
@@ -224,7 +224,6 @@ class TestVPCNics(cloudstackTestCase):
admin=True,
domainid=self.domain.id)
- self.cleanup = [self.account]
self.logger.debug("Creating a VPC offering..")
self.vpc_off = VpcOffering.create(
self.apiclient,
@@ -242,10 +241,13 @@ class TestVPCNics(cloudstackTestCase):
zoneid=self.zone.id,
account=self.account.name,
domainid=self.account.domainid)
+
+ self.cleanup = [self.vpc, self.vpc_off, self.account]
return
def tearDown(self):
try:
+ self.destroy_routers()
cleanup_resources(self.apiclient, self.cleanup)
except Exception as e:
self.logger.debug("Warning: Exception during cleanup : %s" % e)
@@ -261,19 +263,20 @@ class TestVPCNics(cloudstackTestCase):
isinstance(self.routers, list), True,
"Check for list routers response return valid data")
- def stop_router(self):
+ def stop_router(self, router):
self.logger.debug('Stopping router')
- for router in self.routers:
- cmd = stopRouter.stopRouterCmd()
- cmd.id = router.id
- self.apiclient.stopRouter(cmd)
+ cmd = stopRouter.stopRouterCmd()
+ cmd.id = router.id
+ self.apiclient.stopRouter(cmd)
- def destroy_router(self):
- self.logger.debug('Stopping router')
+ def destroy_routers(self):
+ self.logger.debug('Destroying routers')
for router in self.routers:
+ self.stop_router(router)
cmd = destroyRouter.destroyRouterCmd()
cmd.id = router.id
self.apiclient.destroyRouter(cmd)
+ self.routers = []
def create_network(self, net_offerring, gateway='10.1.1.1', vpc=None):
try:
@@ -299,12 +302,19 @@ class TestVPCNics(cloudstackTestCase):
gateway=gateway,
vpcid=vpc.id if vpc else self.vpc.id
)
-
+
+
self.logger.debug("Created network with ID: %s" % obj_network.id)
except Exception, e:
self.fail('Unable to create a Network with offering=%s because of %s ' % (net_offerring,
e))
o = networkO(obj_network)
- o.add_vm(self.deployvm_in_network(obj_network))
+
+ vm1 = self.deployvm_in_network(obj_network)
+
+ self.cleanup.insert(1, obj_network)
+ self.cleanup.insert(2, nw_off)
+
+ o.add_vm(vm1)
return o
def deployvm_in_network(self, network):
@@ -318,7 +328,9 @@ class TestVPCNics(cloudstackTestCase):
serviceofferingid=self.service_offering.id,
networkids=[str(network.id)]
)
+
self.logger.debug('Created VM=%s in network=%s' % (vm.id, network.name))
+ self.cleanup.insert(0, vm)
return vm
except:
self.fail('Unable to create VM in a Network=%s' % network.name)
@@ -362,17 +374,6 @@ class TestVPCNics(cloudstackTestCase):
self.logger.debug('nwacl_nat=%s' % nwacl_nat.__dict__)
return nat_rule
- def check_ssh_into_vm(self, vm, public_ip):
- self.logger.debug("Checking if we can SSH into VM=%s on public_ip=%s" %
- (vm.name, public_ip.ipaddress.ipaddress))
- vm.ssh_client = None
- try:
- vm.get_ssh_client(ipaddress=public_ip.ipaddress.ipaddress)
- self.logger.debug("SSH into VM=%s on public_ip=%s is successful" %
- (vm.name, public_ip.ipaddress.ipaddress))
- except:
- self.fail("Failed to SSH into VM - %s" % (public_ip.ipaddress.ipaddress))
-
@attr(tags=["advanced", "intervlan"], required_hardware="true")
def test_01_VPC_nics_after_destroy(self):
""" Create a VPC with two networks with one VM in each network and test nics after
destroy"""
@@ -386,15 +387,16 @@ class TestVPCNics(cloudstackTestCase):
self.networks.append(net2)
self.add_nat_rules()
- self.test_ssh_to_vm()
+ self.check_ssh_into_vm()
- self.stop_router()
- self.destroy_router()
+ self.destroy_routers()
time.sleep(30)
net1.add_vm(self.deployvm_in_network(net1.get_net()))
+ self.query_routers()
+
self.add_nat_rules()
- self.test_ssh_to_vm()
+ self.check_ssh_into_vm()
@attr(tags=["advanced", "intervlan"], required_hardware="true")
def test_02_VPC_default_routes(self):
@@ -409,7 +411,7 @@ class TestVPCNics(cloudstackTestCase):
self.networks.append(net2)
self.add_nat_rules()
- self.test_default_routes()
+ self.do_default_routes_test()
def delete_nat_rules(self):
for o in self.networks:
@@ -427,12 +429,25 @@ class TestVPCNics(cloudstackTestCase):
vm.set_nat(self.create_natrule(vm.get_vm(), vm.get_ip(), o.get_net()))
time.sleep(5)
- def test_ssh_to_vm(self):
+ def check_ssh_into_vm(self):
for o in self.networks:
for vm in o.get_vms():
- self.check_ssh_into_vm(vm.get_vm(), vm.get_ip())
+ try:
+ virtual_machine = vm.get_vm()
+ virtual_machine.ssh_client = None
+
+ public_ip = vm.get_ip()
+
+ self.logger.debug("Checking if we can SSH into VM=%s on public_ip=%s"
%
+ (virtual_machine.name, public_ip.ipaddress.ipaddress))
+
+ virtual_machine.get_ssh_client(ipaddress=public_ip.ipaddress.ipaddress)
+ self.logger.debug("SSH into VM=%s on public_ip=%s is successful" %
+ (virtual_machine.name, public_ip.ipaddress.ipaddress))
+ except:
+ self.fail("Failed to SSH into VM - %s" % (public_ip.ipaddress.ipaddress))
- def test_default_routes(self):
+ def do_default_routes_test(self):
for o in self.networks:
for vmObj in o.get_vms():
ssh_command = "ping -c 3 8.8.8.8"
@@ -449,7 +464,7 @@ class TestVPCNics(cloudstackTestCase):
self.logger.debug("Ping to google.com from VM")
result = str(ssh.execute(ssh_command))
- self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("0%
packet loss")))
+ self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("3
packets received")))
except Exception as e:
self.fail("SSH Access failed for %s: %s" % \
(vmObj.get_ip(), e)
|