hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-15463 Region normalizer should check whether split/merge is enabled
Date Tue, 15 Mar 2016 16:03:41 GMT
Repository: hbase
Updated Branches:
  refs/heads/master ca816f078 -> 8fcc1e8e9


HBASE-15463 Region normalizer should check whether split/merge is enabled


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8fcc1e8e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8fcc1e8e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8fcc1e8e

Branch: refs/heads/master
Commit: 8fcc1e8e95e9ad063c729de8088fc35a5513b8c2
Parents: ca816f0
Author: tedyu <yuzhihong@gmail.com>
Authored: Tue Mar 15 09:03:27 2016 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Tue Mar 15 09:03:27 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/master/HMaster.java |  1 +
 .../master/normalizer/RegionNormalizer.java     |  8 ++++
 .../normalizer/SimpleRegionNormalizer.java      | 50 +++++++++++++++-----
 .../normalizer/TestSimpleRegionNormalizer.java  | 17 +++++++
 4 files changed, 65 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/8fcc1e8e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 142502a..7942b28 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -583,6 +583,7 @@ public class HMaster extends HRegionServer implements MasterServices {
     this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
     this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
     this.normalizer.setMasterServices(this);
+    this.normalizer.setMasterRpcServices((MasterRpcServices)rpcServices);
     this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
     this.loadBalancerTracker.start();
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/8fcc1e8e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
index 0ee25f4..5b9f592 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.master.MasterRpcServices;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 
@@ -47,6 +48,13 @@ public interface RegionNormalizer {
   void setMasterServices(MasterServices masterServices);
 
   /**
+   * Set the master RPC service. Must be called before first call to
+   * {@link #computePlanForTable(TableName)}.
+   * @param masterRpcServices master RPC services to use
+   */
+  void setMasterRpcServices(MasterRpcServices masterRpcServices);
+
+  /**
    * Computes next optimal normalization plan.
    * @param table table to normalize
    * @return normalization actions to perform. Null if no action to take

http://git-wip-us.apache.org/repos/asf/hbase/blob/8fcc1e8e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index a99f8dd..13b5fab 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -18,6 +18,8 @@
  */
 package org.apache.hadoop.hbase.master.normalizer;
 
+import com.google.protobuf.ServiceException;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseIOException;
@@ -26,8 +28,11 @@ import org.apache.hadoop.hbase.RegionLoad;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin.MasterSwitchType;
+import org.apache.hadoop.hbase.master.MasterRpcServices;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
+import org.apache.hadoop.hbase.protobuf.RequestConverter;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -59,6 +64,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
   private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class);
   private static final int MIN_REGION_COUNT = 3;
   private MasterServices masterServices;
+  private MasterRpcServices masterRpcServices;
   private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
 
   /**
@@ -71,6 +77,11 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
   }
 
   @Override
+  public void setMasterRpcServices(MasterRpcServices masterRpcServices) {
+    this.masterRpcServices = masterRpcServices;
+  }
+
+  @Override
   public void planSkipped(HRegionInfo hri, PlanType type) {
     skippedCount[type.ordinal()]++;
   }
@@ -138,27 +149,44 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
     LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
 
     int candidateIdx = 0;
+    boolean splitEnabled = true, mergeEnabled = true;
+    try {
+      splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+        RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled();
+    } catch (ServiceException se) {
+      LOG.debug("Unable to determine whether split is enabled", se);
+    }
+    try {
+      mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null,
+        RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled();
+    } catch (ServiceException se) {
+      LOG.debug("Unable to determine whether split is enabled", se);
+    }
     while (candidateIdx < tableRegions.size()) {
       HRegionInfo hri = tableRegions.get(candidateIdx);
       long regionSize = getRegionSize(hri);
       // if the region is > 2 times larger than average, we split it, split
       // is more high priority normalization action than merge.
       if (regionSize > 2 * avgRegionSize) {
-        LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + " has
size "
-            + regionSize + ", more than twice avg size, splitting");
-        plans.add(new SplitNormalizationPlan(hri, null));
+        if (splitEnabled) {
+          LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + "
has size "
+              + regionSize + ", more than twice avg size, splitting");
+          plans.add(new SplitNormalizationPlan(hri, null));
+        }
       } else {
         if (candidateIdx == tableRegions.size()-1) {
           break;
         }
-        HRegionInfo hri2 = tableRegions.get(candidateIdx+1);
-        long regionSize2 = getRegionSize(hri2);
-        if (regionSize + regionSize2 < avgRegionSize) {
-          LOG.info("Table " + table + ", small region size: " + regionSize
-            + " plus its neighbor size: " + regionSize2
-            + ", less than the avg size " + avgRegionSize + ", merging them");
-          plans.add(new MergeNormalizationPlan(hri, hri2));
-          candidateIdx++;
+        if (mergeEnabled) {
+          HRegionInfo hri2 = tableRegions.get(candidateIdx+1);
+          long regionSize2 = getRegionSize(hri2);
+          if (regionSize + regionSize2 < avgRegionSize) {
+            LOG.info("Table " + table + ", small region size: " + regionSize
+              + " plus its neighbor size: " + regionSize2
+              + ", less than the avg size " + avgRegionSize + ", merging them");
+            plans.add(new MergeNormalizationPlan(hri, hri2));
+            candidateIdx++;
+          }
         }
       }
       candidateIdx++;

http://git-wip-us.apache.org/repos/asf/hbase/blob/8fcc1e8e/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index babf22d..9d171f0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -25,7 +25,11 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.RegionLoad;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.MasterRpcServices;
 import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.RequestConverter;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -34,6 +38,9 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -56,6 +63,7 @@ public class TestSimpleRegionNormalizer {
 
   // mocks
   private static MasterServices masterServices;
+  private static MasterRpcServices masterRpcServices;
 
   @BeforeClass
   public static void beforeAllTests() throws Exception {
@@ -259,6 +267,7 @@ public class TestSimpleRegionNormalizer {
   protected void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
                                          List<HRegionInfo> hris) {
     masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);
+    masterRpcServices = Mockito.mock(MasterRpcServices.class, RETURNS_DEEP_STUBS);
 
     // for simplicity all regions are assumed to be on one server; doesn't matter to us
     ServerName sn = ServerName.valueOf("localhost", 0, 1L);
@@ -275,7 +284,15 @@ public class TestSimpleRegionNormalizer {
       when(masterServices.getServerManager().getLoad(sn).
         getRegionsLoad().get(region.getKey())).thenReturn(regionLoad);
     }
+    try {
+      when(masterRpcServices.isSplitOrMergeEnabled(any(RpcController.class),
+        any(IsSplitOrMergeEnabledRequest.class))).thenReturn(
+          IsSplitOrMergeEnabledResponse.newBuilder().setEnabled(true).build());
+    } catch (ServiceException se) {
+      LOG.debug("error setting isSplitOrMergeEnabled switch", se);
+    }
 
     normalizer.setMasterServices(masterServices);
+    normalizer.setMasterRpcServices(masterRpcServices);
   }
 }


Mime
View raw message