cloudstack-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (CLOUDSTACK-9595) Transactions are not getting retried in case of database deadlock errors
Date Mon, 18 Dec 2017 18:37:00 GMT

    [ https://issues.apache.org/jira/browse/CLOUDSTACK-9595?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16295397#comment-16295397
] 

ASF GitHub Bot commented on CLOUDSTACK-9595:
--------------------------------------------

rhtyd closed pull request #1762: CLOUDSTACK-9595 Transactions are not getting retried in case
of datab…
URL: https://github.com/apache/cloudstack/pull/1762
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
index f12a62816fc..6cb09b7c5ba 100644
--- a/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
+++ b/engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
@@ -539,7 +539,6 @@ private boolean canOwnCluster(long clusterId) {
     public List<HostVO> findAndUpdateDirectAgentToLoad(long lastPingSecondsAfter, Long
limit, long managementServerId) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
 
-        txn.start();
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("Resetting hosts suitable for reconnect");
         }
@@ -555,6 +554,7 @@ private boolean canOwnCluster(long clusterId) {
             s_logger.debug("Acquiring hosts for clusters already owned by this management
server");
         }
         List<Long> clusters = findClustersOwnedByManagementServer(managementServerId);
+        txn.start();
         if (clusters.size() > 0) {
             // handle clusters already owned by @managementServerId
             SearchCriteria<HostVO> sc = UnmanagedDirectConnectSearch.create();
diff --git a/server/src/com/cloud/network/IpAddressManagerImpl.java b/server/src/com/cloud/network/IpAddressManagerImpl.java
index f3584d1b650..2785df160ec 100644
--- a/server/src/com/cloud/network/IpAddressManagerImpl.java
+++ b/server/src/com/cloud/network/IpAddressManagerImpl.java
@@ -291,6 +291,8 @@
 
     SearchBuilder<IPAddressVO> AssignIpAddressSearch;
     SearchBuilder<IPAddressVO> AssignIpAddressFromPodVlanSearch;
+    private final Object _allocatedLock = new Object();
+    private final Object _allocatingLock = new Object();
 
     static Boolean rulesContinueOnErrFlag = true;
 
@@ -759,7 +761,7 @@ public IPAddressVO doInTransaction(TransactionStatus status) throws Insufficient
 
                 Filter filter = new Filter(IPAddressVO.class, "vlanId", true, 0l, 1l);
 
-                List<IPAddressVO> addrs = _ipAddressDao.lockRows(sc, filter, true);
+                List<IPAddressVO> addrs = _ipAddressDao.search(sc, filter, false);
 
                 // If all the dedicated IPs of the owner are in use fetch an IP from the
system pool
                 if (addrs.size() == 0 && fetchFromDedicatedRange) {
@@ -769,7 +771,7 @@ public IPAddressVO doInTransaction(TransactionStatus status) throws Insufficient
                         fetchFromDedicatedRange = false;
                         sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray());
                         errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray()));
-                        addrs = _ipAddressDao.lockRows(sc, filter, true);
+                        addrs = _ipAddressDao.search(sc, filter, false);
                     }
                 }
 
@@ -804,24 +806,21 @@ public IPAddressVO doInTransaction(TransactionStatus status) throws
Insufficient
                 addr.setAllocatedInDomainId(owner.getDomainId());
                 addr.setAllocatedToAccountId(owner.getId());
                 addr.setSystem(isSystem);
+
                 if (displayIp != null) {
                     addr.setDisplay(displayIp);
                 }
 
-                if (assign) {
-                    markPublicIpAsAllocated(addr);
-                } else {
-                    addr.setState(IpAddress.State.Allocating);
-                }
-                addr.setState(assign ? IpAddress.State.Allocated : IpAddress.State.Allocating);
-
                 if (vlanUse != VlanType.DirectAttached) {
                     addr.setAssociatedWithNetworkId(guestNetworkId);
                     addr.setVpcId(vpcId);
                 }
 
-                _ipAddressDao.update(addr.getId(), addr);
-
+                if (assign) {
+                    markPublicIpAsAllocated(addr);
+                } else {
+                    markPublicIpAsAllocating(addr);
+                }
                 return addr;
             }
         });
@@ -836,35 +835,51 @@ public IPAddressVO doInTransaction(TransactionStatus status) throws
Insufficient
     @DB
     @Override
     public void markPublicIpAsAllocated(final IPAddressVO addr) {
-
-        Transaction.execute(new TransactionCallbackNoReturn() {
-            @Override
-            public void doInTransactionWithoutResult(TransactionStatus status) {
-                Account owner = _accountMgr.getAccount(addr.getAllocatedToAccountId());
-                synchronized (this) {
+        synchronized (_allocatedLock) {
+            Transaction.execute(new TransactionCallbackNoReturn() {
+                @Override
+                public void doInTransactionWithoutResult(TransactionStatus status) {
+                    Account owner = _accountMgr.getAccount(addr.getAllocatedToAccountId());
                     if (_ipAddressDao.lockRow(addr.getId(), true) != null) {
                         IPAddressVO userIp = _ipAddressDao.findById(addr.getId());
                         if (userIp.getState() == IpAddress.State.Allocating || addr.getState()
== IpAddress.State.Free) {
                             addr.setState(IpAddress.State.Allocated);
-                            _ipAddressDao.update(addr.getId(), addr);
-                            // Save usage event
-                            if (owner.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
-                                VlanVO vlan = _vlanDao.findById(addr.getVlanId());
-                                String guestType = vlan.getVlanType().toString();
-                                if (!isIpDedicated(addr)) {
-                                    UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN,
owner.getId(), addr.getDataCenterId(), addr.getId(),
-                                            addr.getAddress().toString(),
-                                            addr.isSourceNat(), guestType, addr.getSystem(),
addr.getClass().getName(), addr.getUuid());
-                                }
-                                if (updateIpResourceCount(addr)) {
-                                    _resourceLimitMgr.incrementResourceCount(owner.getId(),
ResourceType.public_ip);
+                            if (_ipAddressDao.update(addr.getId(), addr)) {
+                                // Save usage event
+                                if (owner.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
+                                    VlanVO vlan = _vlanDao.findById(addr.getVlanId());
+                                    String guestType = vlan.getVlanType().toString();
+                                    if (!isIpDedicated(addr)) {
+                                        UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN,
owner.getId(), addr.getDataCenterId(), addr.getId(),
+                                                addr.getAddress().toString(),
+                                                addr.isSourceNat(), guestType, addr.getSystem(),
addr.getClass().getName(), addr.getUuid());
+                                    }
+                                    if (updateIpResourceCount(addr)) {
+                                        _resourceLimitMgr.incrementResourceCount(owner.getId(),
ResourceType.public_ip);
+                                    }
                                 }
                             }
                         }
                     }
                 }
-            }
-        });
+            });
+        }
+    }
+
+    @DB
+    private void markPublicIpAsAllocating(final IPAddressVO addr) {
+        synchronized (_allocatingLock) {
+            Transaction.execute(new TransactionCallbackNoReturn() {
+                @Override
+                public void doInTransactionWithoutResult(TransactionStatus status) {
+
+                    if (_ipAddressDao.lockRow(addr.getId(), true) != null) {
+                        addr.setState(IpAddress.State.Allocating);
+                        _ipAddressDao.update(addr.getId(), addr);
+                    }
+                }
+            });
+        }
     }
 
     private boolean isIpDedicated(IPAddressVO addr) {
@@ -901,40 +916,28 @@ public PublicIp assignDedicateIpAddress(Account owner, final Long guestNtwkId,
f
 
         PublicIp ip = null;
         try {
-            ip = Transaction.execute(new TransactionCallbackWithException<PublicIp, InsufficientAddressCapacityException>()
{
-                @Override
-                public PublicIp doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException
{
-                    Account owner = _accountDao.acquireInLockTable(ownerId);
-
-                    if (owner == null) {
-                        // this ownerId comes from owner or type Account. See the class "AccountVO"
and the annotations in that class
-                        // to get the table name and field name that is queried to fill this
ownerid.
-                        ConcurrentOperationException ex = new ConcurrentOperationException("Unable
to lock account");
-                        throw ex;
-                    }
-                    if (s_logger.isDebugEnabled()) {
-                        s_logger.debug("lock account " + ownerId + " is acquired");
-                    }
-                    boolean displayIp = true;
-                    if (guestNtwkId != null) {
-                        Network ntwk = _networksDao.findById(guestNtwkId);
-                        displayIp = ntwk.getDisplayNetwork();
-                    } else if (vpcId != null) {
-                        VpcVO vpc = _vpcDao.findById(vpcId);
-                        displayIp = vpc.isDisplay();
-                    }
+            Account ownerAccount = _accountDao.acquireInLockTable(ownerId);
 
-                    PublicIp ip = fetchNewPublicIp(dcId, null, null, owner, VlanType.VirtualNetwork,
guestNtwkId, isSourceNat, false, null, false, vpcId, displayIp);
-                    IPAddressVO publicIp = ip.ip();
-
-                    markPublicIpAsAllocated(publicIp);
-                    _ipAddressDao.update(publicIp.getId(), publicIp);
+            if (ownerAccount == null) {
+                // this ownerId comes from owner or type Account. See the class "AccountVO"
and the annotations in that class
+                // to get the table name and field name that is queried to fill this ownerid.
+                ConcurrentOperationException ex = new ConcurrentOperationException("Unable
to lock account");
+                throw ex;
+            }
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("lock account " + ownerId + " is acquired");
+            }
+            boolean displayIp = true;
+            if (guestNtwkId != null) {
+                Network ntwk = _networksDao.findById(guestNtwkId);
+                displayIp = ntwk.getDisplayNetwork();
+            } else if (vpcId != null) {
+                VpcVO vpc = _vpcDao.findById(vpcId);
+                displayIp = vpc.isDisplay();
+            }
 
-                    return ip;
-                }
-            });
+            return fetchNewPublicIp(dcId, null, null, owner, VlanType.VirtualNetwork, guestNtwkId,
isSourceNat, false, null, false, vpcId, displayIp);
 
-            return ip;
         } finally {
             if (owner != null) {
                 if (s_logger.isDebugEnabled()) {


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Transactions are not getting retried in case of database deadlock errors
> ------------------------------------------------------------------------
>
>                 Key: CLOUDSTACK-9595
>                 URL: https://issues.apache.org/jira/browse/CLOUDSTACK-9595
>             Project: CloudStack
>          Issue Type: Bug
>      Security Level: Public(Anyone can view this level - this is the default.) 
>    Affects Versions: 4.8.0
>            Reporter: subhash yedugundla
>             Fix For: 4.8.1
>
>
> Customer is seeing occasional error 'Deadlock found when trying to get lock; try restarting
transaction' messages in their management server logs.  It happens regularly at least once
a day.  The following is the error seen 
> 2015-12-09 19:23:19,450 ERROR [cloud.api.ApiServer] (catalina-exec-3:ctx-f05c58fc ctx-39c17156
ctx-7becdf6e) unhandled exception executing api command: [Ljava.lang.String;@230a6e7f
> com.cloud.utils.exception.CloudRuntimeException: DB Exception on: com.mysql.jdbc.JDBC4PreparedStatement@74f134e3:
DELETE FROM instance_group_vm_map WHERE instance_group_vm_map.instance_id = 941374
> 	at com.cloud.utils.db.GenericDaoBase.expunge(GenericDaoBase.java:1209)
> 	at sun.reflect.GeneratedMethodAccessor360.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:606)
> 	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
> 	at com.cloud.utils.db.TransactionContextInterceptor.invoke(TransactionContextInterceptor.java:34)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:161)
> 	at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
> 	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
> 	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
> 	at com.sun.proxy.$Proxy237.expunge(Unknown Source)
> 	at com.cloud.vm.UserVmManagerImpl$2.doInTransactionWithoutResult(UserVmManagerImpl.java:2593)
> 	at com.cloud.utils.db.TransactionCallbackNoReturn.doInTransaction(TransactionCallbackNoReturn.java:25)
> 	at com.cloud.utils.db.Transaction$2.doInTransaction(Transaction.java:57)
> 	at com.cloud.utils.db.Transaction.execute(Transaction.java:45)
> 	at com.cloud.utils.db.Transaction.execute(Transaction.java:54)
> 	at com.cloud.vm.UserVmManagerImpl.addInstanceToGroup(UserVmManagerImpl.java:2575)
> 	at com.cloud.vm.UserVmManagerImpl.updateVirtualMachine(UserVmManagerImpl.java:2332)



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Mime
View raw message