cloudstack-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Chandan Purushothama (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (CLOUDSTACK-8022) [Automation] Deletion of Domain with Cleanup set to true fails
Date Thu, 04 Dec 2014 22:51:12 GMT

     [ https://issues.apache.org/jira/browse/CLOUDSTACK-8022?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Chandan Purushothama updated CLOUDSTACK-8022:
---------------------------------------------
    Attachment: management-server.zip

> [Automation] Deletion of Domain with Cleanup set to true fails
> --------------------------------------------------------------
>
>                 Key: CLOUDSTACK-8022
>                 URL: https://issues.apache.org/jira/browse/CLOUDSTACK-8022
>             Project: CloudStack
>          Issue Type: Bug
>      Security Level: Public(Anyone can view this level - this is the default.) 
>          Components: Automation
>    Affects Versions: 4.5.0
>            Reporter: Chandan Purushothama
>            Priority: Critical
>             Fix For: 4.5.0
>
>         Attachments: management-server.zip
>
>
> The following test case in test_persistent_rules.py fails:
> {code}
>     @attr(tags=["advanced"])
>     def test_vpc_force_delete_domain(self):
>         # steps
>         # 1. Create account and create VPC network in the account
>         # 2. Create two persistent networks within this VPC
>         # 3. Restart/delete VPC network
>         # Validations
>         # 1. In case of Restart operation, restart should be successful
>         #    and persistent networks should be back in persistent state
>         # 2. In case of Delete operation, VR servicing the VPC should
>         #    get destroyed and sourceNAT ip should get released
>         child_domain = Domain.create(self.apiclient,
>                                      services=self.services["domain"],
>                                      parentdomainid=self.domain.id)
>         try:
>             account_1 = Account.create(
>                 self.apiclient, self.services["account"],
>                 domainid=child_domain.id
>             )
>             account_2 = Account.create(
>                 self.apiclient, self.services["account"],
>                 domainid=child_domain.id
>             )
>             self.services["vpc"]["cidr"] = "10.1.1.1/16"
>             vpc_1 = VPC.create(
>                 self.apiclient,
>                 self.services["vpc"],
>                 vpcofferingid=self.vpc_off.id,
>                 zoneid=self.zone.id,
>                 account=account_1.name,
>                 domainid=account_1.domainid)
>             vpcs = VPC.list(self.apiclient, id=vpc_1.id)
>             self.assertEqual(
>                 validateList(vpcs)[0],
>                 PASS,
>                 "VPC list validation failed, vpc list is %s" %
>                 vpcs)
>             vpc_2 = VPC.create(
>                 self.apiclient,
>                 self.services["vpc"],
>                 vpcofferingid=self.vpc_off.id,
>                 zoneid=self.zone.id,
>                 account=account_2.name,
>                 domainid=account_2.domainid)
>             vpcs = VPC.list(self.apiclient, id=vpc_2.id)
>             self.assertEqual(
>                 validateList(vpcs)[0],
>                 PASS,
>                 "VPC list validation failed, vpc list is %s" %
>                 vpcs)
>             persistent_network_1 = Network.create(
>                 self.api_client, self.services["isolated_network"],
>                 networkofferingid=self.persistent_network_offering_NoLB.id,
>                 accountid=account_1.name, domainid=account_1.domainid,
>                 zoneid=self.zone.id, vpcid=vpc_1.id, gateway="10.1.1.1",
>                 netmask="255.255.255.0")
>             response = verifyNetworkState(self.apiclient,
>                                           persistent_network_1.id,
>                                           "implemented"
>                                           )
>             exceptionOccured = response[0]
>             isNetworkInDesiredState = response[1]
>             exceptionMessage = response[2]
>             if (exceptionOccured or (not isNetworkInDesiredState)):
>                 raise Exception(exceptionMessage)
>             self.assertIsNotNone(
>                 persistent_network_1.vlan,
>                 "vlan must not be null for persistent network %s" %
>                 persistent_network_1.id)
>             persistent_network_2 = Network.create(
>                 self.api_client, self.services["isolated_network"],
>                 networkofferingid=self.persistent_network_offering_NoLB.id,
>                 accountid=account_2.name, domainid=account_2.domainid,
>                 zoneid=self.zone.id, vpcid=vpc_2.id, gateway="10.1.1.1",
>                 netmask="255.255.255.0")
>             response = verifyNetworkState(
>                 self.apiclient,
>                 persistent_network_2.id,
>                 "implemented")
>             exceptionOccured = response[0]
>             isNetworkInDesiredState = response[1]
>             exceptionMessage = response[2]
>             if (exceptionOccured or (not isNetworkInDesiredState)):
>                 raise Exception(exceptionMessage)
>             self.assertIsNotNone(
>                 persistent_network_2.vlan,
>                 "vlan must not be null for persistent network: %s" %
>                 persistent_network_2.id)
>         except Exception as e:
>             self.cleanup.append(account_1)
>             self.cleanup.append(account_2)
>             self.cleanup.append(child_domain)
>             self.fail(e)
>         # Force delete domain
>         child_domain.delete(self.apiclient, cleanup=True)
>         self.debug("Waiting for account.cleanup.interval" +
>                    " to cleanup any remaining resouces")
>         # Sleep 3*account.gc to ensure that all resources are deleted
>         wait_for_cleanup(self.apiclient, ["account.cleanup.interval"] * 3)
>         with self.assertRaises(Exception):
>             Domain.list(self.apiclient, id=child_domain.id)
>         with self.assertRaises(Exception):
>             Account.list(
>                 self.apiclient, name=account_1.name,
>                 domainid=account_1.domainid, listall=True
>             )
>         with self.assertRaises(Exception):
>             Account.list(
>                 self.apiclient, name=account_2.name,
>                 domainid=account_2.domainid, listall=True
>             )
>         self.VerifyVpcCleanup(vpc_1.id)
>         self.VerifyVpcCleanup(vpc_2.id)
>         self.VerifyNetworkCleanup(persistent_network_1.id)
>         self.VerifyNetworkCleanup(persistent_network_2.id)
>         return
> {code}
> Following is the stack trace:
> {noformat}
> Stacktrace
>   File "/usr/lib/python2.7/unittest/case.py", line 332, in run
>     testMethod()
>   File "/root/cloudstack/test/integration/component/test_persistent_networks.py", line
2780, in test_vpc_force_delete_domain
>     child_domain.delete(self.apiclient, cleanup=True)
>   File "/usr/local/lib/python2.7/dist-packages/marvin/lib/base.py", line 77, in delete
>     apiclient.deleteDomain(cmd)
>   File "/usr/local/lib/python2.7/dist-packages/marvin/cloudstackAPI/cloudstackAPIClient.py",
line 671, in deleteDomain
>     response = self.connection.marvinRequest(command, response_type=response, method=method)
>   File "/usr/local/lib/python2.7/dist-packages/marvin/cloudstackConnection.py", line
379, in marvinRequest
>     raise e
> 'Job failed: {jobprocstatus : 0, created : u\'2014-12-03T02:53:03+0000\', cmd : u\'org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd\',
userid : u\'cb31c30a-7a84-11e4-bab5-967d6e0d04d3\', jobstatus : 2, jobid : u\'b373c063-dfb6-46bc-b82a-da7f9b12937b\',
jobresultcode : 530, jobresulttype : u\'object\', jobresult : {errorcode : 530, errortext
: u\'Failed to clean up domain resources and sub domains, delete failed on domain domain-RS2LKV
(id: 7).\'}, accountid : u\'cb31b43c-7a84-11e4-bab5-967d6e0d04d3\'}\n
> {noformat}
> ============================================
> Exception Information from the Management Server Log:
> ============================================
> {noformat}
> 2014-12-03 02:54:33,733 DEBUG [c.c.n.v.RemoteAccessVpnManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) there are no Remote access vpns for public ip address
id=12
> 2014-12-03 02:54:33,746 DEBUG [c.c.n.IpAddressManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Releasing ip id=12; sourceNat = true
> 2014-12-03 02:54:33,749 DEBUG [c.c.n.IpAddressManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Released a public ip id=12
> 2014-12-03 02:54:33,749 DEBUG [c.c.n.v.VpcManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Released ip addresses for vpc id=57 as a part of cleanup
vpc process
> 2014-12-03 02:54:33,750 DEBUG [c.c.n.v.VpcManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Found 0 to revoke for the vpc 57
> 2014-12-03 02:54:33,753 DEBUG [c.c.n.v.VpcManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Vpc [VPC [57-vpc_vpn-T13419] is destroyed succesfully
> 2014-12-03 02:54:33,753 DEBUG [c.c.u.AccountManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) VPC 57 successfully deleted as a part of account id=105
cleanup.
> 2014-12-03 02:54:33,754 DEBUG [c.c.u.AccountManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Deleting site-to-site VPN customer gateways for account
105
> 2014-12-03 02:54:33,759 DEBUG [c.c.u.AccountManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Account specific Virtual IP ranges  are successfully
released as a part of account id=105 cleanup.
> 2014-12-03 02:54:33,760 INFO  [c.c.u.AccountManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) deleteAccount: Released 0 dedicated guest vlan ranges
from account 105
> 2014-12-03 02:54:33,806 INFO  [c.c.u.AccountManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Cleanup for account 105 is not needed.
> 2014-12-03 02:54:33,808 DEBUG [c.c.u.DomainManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Deleting networks for domain id=7
> 2014-12-03 02:54:33,812 ERROR [c.c.d.d.DomainDaoImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Unable to remove domain as domain 7 no longer exists
> 2014-12-03 02:54:33,817 ERROR [c.c.u.DomainManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Exception deleting domain with id 7
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain resources
and sub domains, delete failed on domain domain-RS2LKV (id: 7).
> 	at com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
> 	at com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:601)
> 	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> 	at org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> 	at com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> 	at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> 	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> 	at $Proxy114.deleteDomain(Unknown Source)
> 	at org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
> 	at org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
> 	at org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
> 	at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
> 	at com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
> 	at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
> 	at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
> 	at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
> 	at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
> 	at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
> 	at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
> 	at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> 	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> 	at java.lang.Thread.run(Thread.java:722)
> 2014-12-03 02:54:33,819 DEBUG [c.c.u.DomainManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474 ctx-2ac8b66c) (logid:b373c063) Changing domain id=7 state back to Active because
it can't be removed due to resources referencing to it
> 2014-12-03 02:54:33,850 ERROR [c.c.a.ApiAsyncJobDispatcher] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Unexpected exception while executing org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd
> com.cloud.utils.exception.CloudRuntimeException: Failed to clean up domain resources
and sub domains, delete failed on domain domain-RS2LKV (id: 7).
> 	at com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:270)
> 	at com.cloud.user.DomainManagerImpl.deleteDomain(DomainManagerImpl.java:253)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:601)
> 	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> 	at org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:106)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> 	at com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> 	at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> 	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> 	at $Proxy114.deleteDomain(Unknown Source)
> 	at org.apache.cloudstack.region.RegionManagerImpl.deleteDomain(RegionManagerImpl.java:242)
> 	at org.apache.cloudstack.region.RegionServiceImpl.deleteDomain(RegionServiceImpl.java:169)
> 	at org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd.execute(DeleteDomainCmd.java:103)
> 	at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
> 	at com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108)
> 	at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:546)
> 	at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
> 	at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
> 	at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
> 	at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
> 	at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
> 	at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:497)
> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> 	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> 	at java.lang.Thread.run(Thread.java:722)
> 2014-12-03 02:54:33,851 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Complete async job-1474, jobStatus: FAILED, resultCode: 530, result:
org.apache.cloudstack.api.response.ExceptionResponse/null/{"uuidList":[],"errorcode":530,"errortext":"Failed
to clean up domain resources and sub domains, delete failed on domain domain-RS2LKV (id: 7)."}
> 2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Publish async job-1474 complete on message bus
> 2014-12-03 02:54:33,852 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Wake up jobs related to job-1474
> 2014-12-03 02:54:33,853 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Update db status for job-1474
> 2014-12-03 02:54:33,854 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Wake up jobs joined with job-1474 and disjoin all subjobs created
from job- 1474
> 2014-12-03 02:54:33,856 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Done executing org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd
for job-1474
> 2014-12-03 02:54:33,856 INFO  [o.a.c.f.j.i.AsyncJobMonitor] (API-Job-Executor-72:ctx-1a1bd808
job-1474) (logid:b373c063) Remove job-1474 from job monitoring
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message