hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject hbase git commit: HBASE-13635 Regions stuck in transition because master is incorrectly assumed dead
Date Mon, 11 May 2015 22:24:00 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 22dad3775 -> 253782517


HBASE-13635 Regions stuck in transition because master is incorrectly assumed dead

Summary:
Requests to tell master that meta is moved can be blocked behind other requests that are trying
to mutate meta.
This causes a dead lock and the master is assumed dead. However the master stays up the whole
time.

This patch adds prioritization onto reporting meta moves. It should allow meta to make progress.

Test Plan: unit tests.

Differential Revision: https://reviews.facebook.net/D38109


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/25378251
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/25378251
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/25378251

Branch: refs/heads/branch-1.1
Commit: 2537825179a72055bc483b1c1a988b5fc499abf4
Parents: 22dad37
Author: Elliott Clark <eclark@apache.org>
Authored: Wed May 6 11:27:33 2015 -0700
Committer: Elliott Clark <eclark@apache.org>
Committed: Mon May 11 15:22:31 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/ipc/RWQueueRpcExecutor.java    |  3 ++
 .../AnnotationReadingPriorityFunction.java      | 21 ++++++++
 .../hbase/regionserver/TestQosFunction.java     | 52 ++++++++++++++++++++
 3 files changed, 76 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/25378251/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java
index 2b58680..a0ea160 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java
@@ -187,6 +187,9 @@ public class RWQueueRpcExecutor extends RpcExecutor {
     if (methodName.equalsIgnoreCase("mutate")) {
       return true;
     }
+    if (methodName.equalsIgnoreCase("ReportRegionStateTransition")) {
+      return true;
+    }
     return false;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/25378251/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
index 5ee20ef..17c14b6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
@@ -25,9 +25,14 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.ipc.PriorityFunction;
 import org.apache.hadoop.hbase.ipc.QosPriority;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
@@ -227,6 +232,22 @@ class AnnotationReadingPriorityFunction implements PriorityFunction {
         return HConstants.SYSTEMTABLE_QOS;
       }
     }
+
+    // If meta is moving then all the rest of report the report state transitions will be
+    // blocked. We shouldn't be in the same queue.
+    if (methodName.equalsIgnoreCase("ReportRegionStateTransition")) { // Regions are moving
+      ReportRegionStateTransitionRequest tRequest = (ReportRegionStateTransitionRequest)
param;
+      for (RegionStateTransition transition : tRequest.getTransitionList()) {
+        if (transition.getRegionInfoList() != null) {
+          for (HBaseProtos.RegionInfo info : transition.getRegionInfoList()) {
+            TableName tn = ProtobufUtil.toTableName(info.getTableName());
+            if (tn.isSystemTable()) {
+              return HConstants.SYSTEMTABLE_QOS;
+            }
+          }
+        }
+      }
+    }
     return HConstants.NORMAL_QOS;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/25378251/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java
index 13e3378..4d5d7ef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java
@@ -22,9 +22,16 @@ import static org.mockito.Mockito.when;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
 import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
@@ -52,6 +59,51 @@ public class TestQosFunction {
     checkMethod("OpenRegion", HConstants.ADMIN_QOS, qosFunction);
     // Check multi works.
     checkMethod("Multi", HConstants.NORMAL_QOS, qosFunction, MultiRequest.getDefaultInstance());
+
+  }
+
+  @Test
+  public void testRegionInTransition() {
+    Configuration conf = HBaseConfiguration.create();
+    RSRpcServices rpcServices = Mockito.mock(RSRpcServices.class);
+    when(rpcServices.getConfiguration()).thenReturn(conf);
+
+    AnnotationReadingPriorityFunction qosFunction =
+        new AnnotationReadingPriorityFunction(rpcServices, RSRpcServices.class);
+
+    // Check ReportRegionInTransition
+    HBaseProtos.RegionInfo meta_ri = HRegionInfo.convert(HRegionInfo.FIRST_META_REGIONINFO);
+    HBaseProtos.RegionInfo normal_ri = HRegionInfo.convert(
+        new HRegionInfo(TableName.valueOf("test:table"),
+            Bytes.toBytes("a"), Bytes.toBytes("b"), false));
+
+
+    RegionServerStatusProtos.RegionStateTransition metaTransition = RegionServerStatusProtos
+        .RegionStateTransition.newBuilder()
+        .addRegionInfo(meta_ri)
+        .setTransitionCode(RegionServerStatusProtos.RegionStateTransition.TransitionCode.CLOSED)
+        .build();
+
+    RegionServerStatusProtos.RegionStateTransition normalTransition = RegionServerStatusProtos
+        .RegionStateTransition.newBuilder()
+        .addRegionInfo(normal_ri)
+        .setTransitionCode(RegionServerStatusProtos.RegionStateTransition.TransitionCode.CLOSED)
+        .build();
+
+    RegionServerStatusProtos.ReportRegionStateTransitionRequest metaTransitionRequest =
+        RegionServerStatusProtos.ReportRegionStateTransitionRequest.newBuilder()
+            .setServer(ProtobufUtil.toServerName(ServerName.valueOf("locahost:60020", 100)))
+            .addTransition(normalTransition)
+            .addTransition(metaTransition).build();
+
+    RegionServerStatusProtos.ReportRegionStateTransitionRequest normalTransitionRequest =
+        RegionServerStatusProtos.ReportRegionStateTransitionRequest.newBuilder()
+            .setServer(ProtobufUtil.toServerName(ServerName.valueOf("locahost:60020", 100)))
+            .addTransition(normalTransition).build();
+
+    final String reportFuncName = "ReportRegionStateTransition";
+    checkMethod(reportFuncName, HConstants.SYSTEMTABLE_QOS, qosFunction, metaTransitionRequest);
+    checkMethod(reportFuncName, HConstants.NORMAL_QOS, qosFunction, normalTransitionRequest);
   }
 
   private void checkMethod(final String methodName, final int expected,


Mime
View raw message