cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kelv...@apache.org
Subject [2/3] git commit: updated refs/heads/master to cd8801f
Date Wed, 02 Apr 2014 22:47:28 GMT
BUG-ID: CS-19697: Deal with possible DB deadlock issue, return the job into queue for next
turn


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/66486d43
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/66486d43
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/66486d43

Branch: refs/heads/master
Commit: 66486d4322121c2a8903ab94db60fb894c5ad149
Parents: 66bece5
Author: Kelven Yang <kelveny@gmail.com>
Authored: Mon Mar 24 17:50:27 2014 -0700
Committer: Kelven Yang <kelveny@gmail.com>
Committed: Wed Apr 2 14:59:04 2014 -0700

----------------------------------------------------------------------
 .../framework/messagebus/MessageDetector.java   |  4 ++-
 .../jobs/impl/AsyncJobManagerImpl.java          | 38 ++++++++++++++++----
 2 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/66486d43/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
----------------------------------------------------------------------
diff --git a/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
index 3fb620c..fae9bf3 100644
--- a/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
+++ b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java
@@ -31,8 +31,10 @@ public class MessageDetector implements MessageSubscriber {
     }
 
     public boolean waitAny(long timeoutInMiliseconds) {
-        _signalled = false;
         synchronized (this) {
+            if (_signalled)
+                return true;
+
             try {
                 wait(timeoutInMiliseconds);
             } catch (InterruptedException e) {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/66486d43/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
index 4acfdbf..42148be 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
@@ -445,9 +445,9 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
                     CallContext.registerPlaceHolderContext();
 
                 if (job.getRelated() != null && !job.getRelated().isEmpty())
-                    NDC.push("Job-" + job.getRelated() + "/" + "Job-" + job.getId());
+                    NDC.push("job-" + job.getRelated() + "/" + "job-" + job.getId());
                 else
-                    NDC.push("Job-" + job.getId());
+                    NDC.push("job-" + job.getId());
                 try {
                     super.run();
                 } finally {
@@ -560,17 +560,41 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
 
             job.setSyncSource(item);
 
-            job.setExecutingMsid(getMsid());
-            _jobDao.update(job.getId(), job);
+            //
+            // TODO: a temporary solution to work-around DB deadlock situation
+            //
+            // to live with DB deadlocks, we will give a chance for job to be rescheduled
+            // in case of exceptions (most-likely DB deadlock exceptions)
+            try {
+                job.setExecutingMsid(getMsid());
+                _jobDao.update(job.getId(), job);
+            } catch (Exception e) {
+                s_logger.warn("Unexpected exception while dispatching job-" + item.getContentId(),
e);
+
+                try {
+                    _queueMgr.returnItem(item.getId());
+                } catch (Throwable thr) {
+                    s_logger.error("Unexpected exception while returning job-" + item.getContentId()
+ " to queue", thr);
+                }
+            }
 
             try {
                 scheduleExecution(job);
             } catch (RejectedExecutionException e) {
                 s_logger.warn("Execution for job-" + job.getId() + " is rejected, return
it to the queue for next turn");
-                _queueMgr.returnItem(item.getId());
 
-                job.setExecutingMsid(null);
-                _jobDao.update(job.getId(), job);
+                try {
+                    _queueMgr.returnItem(item.getId());
+                } catch (Exception e2) {
+                    s_logger.error("Unexpected exception while returning job-" + item.getContentId()
+ " to queue", e2);
+                }
+
+                try {
+                    job.setExecutingMsid(null);
+                    _jobDao.update(job.getId(), job);
+                } catch (Exception e3) {
+                    s_logger.warn("Unexpected exception while update job-" + item.getContentId()
+ " msid for bookkeeping");
+                }
             }
 
         } else {


Mime
View raw message