hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhang...@apache.org
Subject hbase git commit: HBASE-21376 Add some verbose log to MasterProcedureScheduler
Date Sat, 10 Nov 2018 09:41:43 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-2 ee19f2b36 -> f1d9f59bf


HBASE-21376 Add some verbose log to MasterProcedureScheduler


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f1d9f59b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f1d9f59b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f1d9f59b

Branch: refs/heads/branch-2
Commit: f1d9f59bfe88832cb2e5fb23f83d8bf73314e770
Parents: ee19f2b
Author: Duo Zhang <zhangduo@apache.org>
Authored: Thu Nov 1 15:08:43 2018 +0800
Committer: zhangduo <zhangduo@apache.org>
Committed: Sat Nov 10 17:17:24 2018 +0800

----------------------------------------------------------------------
 .../procedure/MasterProcedureScheduler.java     | 72 ++++++++++++++------
 1 file changed, 50 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/f1d9f59b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index d9a520b..4e60214 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableName;
@@ -156,9 +157,18 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
     // which means it can be executed immediately.
     // 2. The exclusive lock for this queue has not been held.
     // 3. The given procedure has the exclusive lock permission for this queue.
-    if (proc.hasLock() || proc.isLockedWhenLoading() || !queue.getLockStatus().hasExclusiveLock()
||
-      queue.getLockStatus().hasLockAccess(proc)) {
-      addToRunQueue(fairq, queue);
+    Supplier<String> reason = null;
+    if (proc.hasLock()) {
+      reason = () -> proc + " has lock";
+    } else if (proc.isLockedWhenLoading()) {
+      reason = () -> proc + " restores lock when restarting";
+    } else if (!queue.getLockStatus().hasExclusiveLock()) {
+      reason = () -> "the exclusive lock is not held by anyone when adding " + proc;
+    } else if (queue.getLockStatus().hasLockAccess(proc)) {
+      reason = () -> proc + " has the excusive lock access";
+    }
+    if (reason != null) {
+      addToRunQueue(fairq, queue, reason);
     }
   }
 
@@ -211,7 +221,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
       if (isLockReady(proc, rq)) {
         // the queue is empty, remove from run queue
         if (rq.isEmpty()) {
-          removeFromRunQueue(fairq, rq);
+          removeFromRunQueue(fairq, rq, () -> "queue is empty after polling out " + proc);
         }
         return proc;
       }
@@ -219,7 +229,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
       rq.add(proc, false);
     }
     // no procedure is ready for execution, remove from run queue
-    removeFromRunQueue(fairq, rq);
+    removeFromRunQueue(fairq, rq, () -> "no procedure can be executed");
     return null;
   }
 
@@ -273,11 +283,13 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
   }
 
   private <T extends Comparable<T>, TNode extends Queue<T>> void clear(TNode
treeMap,
-      final FairQueue<T> fairq, final AvlKeyComparator<TNode> comparator) {
+      FairQueue<T> fairq, AvlKeyComparator<TNode> comparator) {
     while (treeMap != null) {
       Queue<T> node = AvlTree.getFirst(treeMap);
       treeMap = AvlTree.remove(treeMap, node.getKey(), comparator);
-      if (fairq != null) removeFromRunQueue(fairq, node);
+      if (fairq != null) {
+        removeFromRunQueue(fairq, node, () -> "clear all queues");
+      }
     }
   }
 
@@ -334,14 +346,21 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
     }
   }
 
-  private static <T extends Comparable<T>> void addToRunQueue(FairQueue<T>
fairq, Queue<T> queue) {
+  private static <T extends Comparable<T>> void addToRunQueue(FairQueue<T>
fairq, Queue<T> queue,
+      Supplier<String> reason) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Add {} to run queue because: {}", queue, reason.get());
+    }
     if (!AvlIterableList.isLinked(queue) && !queue.isEmpty()) {
       fairq.add(queue);
     }
   }
 
-  private static <T extends Comparable<T>> void removeFromRunQueue(
-      FairQueue<T> fairq, Queue<T> queue) {
+  private static <T extends Comparable<T>> void removeFromRunQueue(FairQueue<T>
fairq,
+      Queue<T> queue, Supplier<String> reason) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Remove {} from run queue because: {}", queue, reason.get());
+    }
     if (AvlIterableList.isLinked(queue)) {
       fairq.remove(queue);
     }
@@ -411,7 +430,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
 
       LockAndQueue lock = locking.getServerLock(serverName);
       if (node.isEmpty() && lock.tryExclusiveLock(proc)) {
-        removeFromRunQueue(serverRunQueue, node);
+        removeFromRunQueue(serverRunQueue, node,
+          () -> "clean up server queue after " + proc + " completed");
         removeServerQueue(serverName);
       }
     } finally {
@@ -459,7 +479,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
 
       final LockAndQueue lock = locking.getPeerLock(peerId);
       if (queue.isEmpty() && lock.tryExclusiveLock(procedure)) {
-        removeFromRunQueue(peerRunQueue, queue);
+        removeFromRunQueue(peerRunQueue, queue,
+          () -> "clean up peer queue after " + procedure + " completed");
         removePeerQueue(peerId);
       }
     } finally {
@@ -539,7 +560,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
         logLockedResource(LockedResourceType.TABLE, table.getNameAsString());
         return true;
       }
-      removeFromRunQueue(tableRunQueue, getTableQueue(table));
+      removeFromRunQueue(tableRunQueue, getTableQueue(table),
+        () -> procedure + " held the exclusive lock");
       return false;
     } finally {
       schedUnlock();
@@ -563,7 +585,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
       if (namespaceLock.releaseSharedLock()) {
         waitingCount += wakeWaitingProcedures(namespaceLock);
       }
-      addToRunQueue(tableRunQueue, getTableQueue(table));
+      addToRunQueue(tableRunQueue, getTableQueue(table),
+        () -> procedure + " released the exclusive lock");
       wakePollIfNeeded(waitingCount);
     } finally {
       schedUnlock();
@@ -615,7 +638,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
       final LockAndQueue tableLock = locking.getTableLock(table);
       int waitingCount = 0;
       if (tableLock.releaseSharedLock()) {
-        addToRunQueue(tableRunQueue, getTableQueue(table));
+        addToRunQueue(tableRunQueue, getTableQueue(table),
+          () -> procedure + " released the shared lock");
         waitingCount += wakeWaitingProcedures(tableLock);
       }
       if (namespaceLock.releaseSharedLock()) {
@@ -823,7 +847,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
         waitingCount += wakeWaitingProcedures(namespaceLock);
       }
       if (systemNamespaceTableLock.releaseSharedLock()) {
-        addToRunQueue(tableRunQueue, getTableQueue(TableName.NAMESPACE_TABLE_NAME));
+        addToRunQueue(tableRunQueue, getTableQueue(TableName.NAMESPACE_TABLE_NAME),
+          () -> procedure + " released namespace exclusive lock");
         waitingCount += wakeWaitingProcedures(systemNamespaceTableLock);
       }
       wakePollIfNeeded(waitingCount);
@@ -853,7 +878,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
         removeFromRunQueue(serverRunQueue,
           getServerQueue(serverName,
             procedure instanceof ServerProcedureInterface ? (ServerProcedureInterface) procedure
-              : null));
+              : null),
+          () -> procedure + " held exclusive lock");
         return false;
       }
       waitProcedure(lock, procedure);
@@ -881,7 +907,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
       addToRunQueue(serverRunQueue,
         getServerQueue(serverName,
           procedure instanceof ServerProcedureInterface ? (ServerProcedureInterface) procedure
-            : null));
+            : null), () -> procedure + " released exclusive lock");
       int waitingCount = wakeWaitingProcedures(lock);
       wakePollIfNeeded(waitingCount);
     } finally {
@@ -908,7 +934,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
     try {
       final LockAndQueue lock = locking.getPeerLock(peerId);
       if (lock.tryExclusiveLock(procedure)) {
-        removeFromRunQueue(peerRunQueue, getPeerQueue(peerId));
+        removeFromRunQueue(peerRunQueue, getPeerQueue(peerId),
+          () -> procedure + " held exclusive lock");
         return false;
       }
       waitProcedure(lock, procedure);
@@ -930,7 +957,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
     try {
       final LockAndQueue lock = locking.getPeerLock(peerId);
       if (lock.releaseExclusiveLock(procedure)) {
-        addToRunQueue(peerRunQueue, getPeerQueue(peerId));
+        addToRunQueue(peerRunQueue, getPeerQueue(peerId),
+          () -> procedure + " released exclusive lock");
         int waitingCount = wakeWaitingProcedures(lock);
         wakePollIfNeeded(waitingCount);
       }
@@ -956,7 +984,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
     try {
       final LockAndQueue lock = locking.getMetaLock();
       if (lock.tryExclusiveLock(procedure)) {
-        removeFromRunQueue(metaRunQueue, getMetaQueue());
+        removeFromRunQueue(metaRunQueue, getMetaQueue(), () -> procedure + " held exclusive
lock");
         return false;
       }
       waitProcedure(lock, procedure);
@@ -980,7 +1008,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
     try {
       final LockAndQueue lock = locking.getMetaLock();
       lock.releaseExclusiveLock(procedure);
-      addToRunQueue(metaRunQueue, getMetaQueue());
+      addToRunQueue(metaRunQueue, getMetaQueue(), () -> procedure + " released exclusive
lock");
       int waitingCount = wakeWaitingProcedures(lock);
       wakePollIfNeeded(waitingCount);
     } finally {


Mime
View raw message