hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-15065 SimpleRegionNormalizer should return multiple normalization plans in one run
Date Fri, 08 Jan 2016 20:19:26 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 f9c69893e -> a5438f4a5


HBASE-15065 SimpleRegionNormalizer should return multiple normalization plans in one run


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a5438f4a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a5438f4a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a5438f4a

Branch: refs/heads/branch-1.2
Commit: a5438f4a55bc6ff56a1eded712c567141f679600
Parents: f9c6989
Author: tedyu <yuzhihong@gmail.com>
Authored: Fri Jan 8 12:19:19 2016 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Fri Jan 8 12:19:19 2016 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/master/HMaster.java |  8 +-
 .../master/normalizer/RegionNormalizer.java     |  4 +-
 .../normalizer/RegionNormalizerChore.java       |  2 +-
 .../normalizer/SimpleRegionNormalizer.java      | 87 +++++++++-----------
 .../normalizer/TestSimpleRegionNormalizer.java  | 27 +++---
 .../TestSimpleRegionNormalizerOnCluster.java    | 18 ++--
 6 files changed, 75 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/a5438f4a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c51156b..0303618 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -112,6 +112,7 @@ import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
 import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.normalizer.NormalizationPlan;
 import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType;
 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
@@ -1351,7 +1352,12 @@ public class HMaster extends HRegionServer implements MasterServices,
Server {
             continue;
           }
         }
-        this.normalizer.computePlanForTable(table, types).execute(clusterConnection.getAdmin());
+        List<NormalizationPlan> plans = this.normalizer.computePlanForTable(table,
types);
+        if (plans != null) {
+          for (NormalizationPlan plan : plans) {
+            plan.execute(clusterConnection.getAdmin());
+          }
+        }
       }
     }
     // If Region did not generate any plans, it means the cluster is already balanced.

http://git-wip-us.apache.org/repos/asf/hbase/blob/a5438f4a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
index 9de376a..3c5de18 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
@@ -50,8 +50,8 @@ public interface RegionNormalizer {
    * Computes next optimal normalization plan.
    * @param table table to normalize
    * @param types desired types of NormalizationPlan
-   * @return Next (perhaps most urgent) normalization action to perform
+   * @return normalization actions to perform. Null if no action to take
    */
-  NormalizationPlan computePlanForTable(TableName table, List<PlanType> types)
+  List<NormalizationPlan> computePlanForTable(TableName table, List<PlanType>
types)
       throws HBaseIOException;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/a5438f4a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
index 1506892..9f47cdc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java
@@ -39,7 +39,7 @@ public class RegionNormalizerChore extends ScheduledChore {
 
   public RegionNormalizerChore(HMaster master) {
     super(master.getServerName() + "-RegionNormalizerChore", master,
-      master.getConfiguration().getInt("hbase.normalizer.period", 1800000));
+      master.getConfiguration().getInt("hbase.normalizer.period", 300000));
     this.master = master;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/a5438f4a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 9f76f26..3b6981e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -71,19 +71,14 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
     this.masterServices = masterServices;
   }
 
-  /*
-   * This comparator compares the region size.
-   * The second element in the triple is region size while the 3rd element
-   * is the index of the region in the underlying List
-   */
-  private Comparator<Triple<HRegionInfo, Long, Integer>> regionSizeComparator
=
-      new Comparator<Triple<HRegionInfo, Long, Integer>>() {
+  // Comparator that gives higher priority to region Split plan
+  private Comparator<NormalizationPlan> planComparator =
+      new Comparator<NormalizationPlan>() {
     @Override
-    public int compare(Triple<HRegionInfo, Long, Integer> pair,
-        Triple<HRegionInfo, Long, Integer> pair2) {
-      long sz = pair.getSecond();
-      long sz2 = pair2.getSecond();
-      return (sz < sz2) ? -1 : ((sz == sz2) ? 0 : 1);
+    public int compare(NormalizationPlan plan, NormalizationPlan plan2) {
+      if (plan instanceof SplitNormalizationPlan) return -1;
+      if (plan2 instanceof SplitNormalizationPlan) return 1;
+      return 0;
     }
   };
 
@@ -96,13 +91,14 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
    * @return normalization plan to execute
    */
   @Override
-  public NormalizationPlan computePlanForTable(TableName table, List<PlanType> types)
+  public List<NormalizationPlan> computePlanForTable(TableName table, List<PlanType>
types)
       throws HBaseIOException {
     if (table == null || table.isSystemTable()) {
       LOG.debug("Normalization of system table " + table + " isn't allowed");
-      return EmptyNormalizationPlan.getInstance();
+      return null;
     }
 
+    List<NormalizationPlan> plans = new ArrayList<NormalizationPlan>();
     List<HRegionInfo> tableRegions = masterServices.getAssignmentManager().getRegionStates().
       getRegionsOfTable(table);
 
@@ -111,7 +107,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
       int nrRegions = tableRegions == null ? 0 : tableRegions.size();
       LOG.debug("Table " + table + " has " + nrRegions + " regions, required min number"
         + " of regions for normalizer to run is " + MIN_REGION_COUNT + ", not running normalizer");
-      return EmptyNormalizationPlan.getInstance();
+      return null;
     }
 
     LOG.debug("Computing normalization plan for table: " + table +
@@ -119,56 +115,49 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
 
     long totalSizeMb = 0;
 
-    ArrayList<Triple<HRegionInfo, Long, Integer>> regionsWithSize =
-        new ArrayList<Triple<HRegionInfo, Long, Integer>>(tableRegions.size());
     for (int i = 0; i < tableRegions.size(); i++) {
       HRegionInfo hri = tableRegions.get(i);
       long regionSize = getRegionSize(hri);
-      regionsWithSize.add(new Triple<HRegionInfo, Long, Integer>(hri, regionSize, i));
       totalSizeMb += regionSize;
     }
-    Collections.sort(regionsWithSize, regionSizeComparator);
-
-    Triple<HRegionInfo, Long, Integer> largestRegion = regionsWithSize.get(tableRegions.size()-1);
 
     double avgRegionSize = totalSizeMb / (double) tableRegions.size();
 
     LOG.debug("Table " + table + ", total aggregated regions size: " + totalSizeMb);
     LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
 
-    // now; if the largest region is >2 times large than average, we split it, split
-    // is more high priority normalization action than merge.
-    if (types.contains(PlanType.SPLIT) && largestRegion.getSecond() > 2 * avgRegionSize)
{
-      LOG.debug("Table " + table + ", largest region "
-        + largestRegion.getFirst().getRegionNameAsString() + " has size "
-        + largestRegion.getSecond() + ", more than 2 times than avg size, splitting");
-      return new SplitNormalizationPlan(largestRegion.getFirst(), null);
-    }
     int candidateIdx = 0;
-    // look for two successive entries whose indices are adjacent
-    while (candidateIdx < tableRegions.size()-1) {
-      if (Math.abs(regionsWithSize.get(candidateIdx).getThird() -
-        regionsWithSize.get(candidateIdx + 1).getThird()) == 1) {
-        break;
+    while (candidateIdx < tableRegions.size()) {
+      HRegionInfo hri = tableRegions.get(candidateIdx);
+      long regionSize = getRegionSize(hri);
+      // if the region is > 2 times larger than average, we split it, split
+      // is more high priority normalization action than merge.
+      if (types.contains(PlanType.SPLIT) && regionSize > 2 * avgRegionSize) {
+        LOG.debug("Table " + table + ", large region " + hri.getRegionNameAsString() + "
has size "
+            + regionSize + ", more than twice avg size, splitting");
+        plans.add(new SplitNormalizationPlan(hri, null));
+      } else {
+        if (candidateIdx == tableRegions.size()-1) {
+          break;
+        }
+        HRegionInfo hri2 = tableRegions.get(candidateIdx+1);
+        long regionSize2 = getRegionSize(hri2);
+        if (types.contains(PlanType.MERGE) && regionSize + regionSize2 < avgRegionSize)
{
+          LOG.debug("Table " + table + ", small region size: " + regionSize
+            + " plus its neighbor size: " + regionSize2
+            + ", less than the avg size " + avgRegionSize + ", merging them");
+          plans.add(new MergeNormalizationPlan(hri, hri2));
+          candidateIdx++;
+        }
       }
       candidateIdx++;
     }
-    if (candidateIdx == tableRegions.size()-1) {
-      LOG.debug("No neighboring regions found for table: " + table);
-      return EmptyNormalizationPlan.getInstance();
-    }
-    Triple<HRegionInfo, Long, Integer> candidateRegion = regionsWithSize.get(candidateIdx);
-    Triple<HRegionInfo, Long, Integer> candidateRegion2 = regionsWithSize.get(candidateIdx+1);
-    if (types.contains(PlanType.MERGE) &&
-        candidateRegion.getSecond() + candidateRegion2.getSecond() < avgRegionSize) {
-      LOG.debug("Table " + table + ", smallest region size: " + candidateRegion.getSecond()
-        + " and its smallest neighbor size: " + candidateRegion2.getSecond()
-        + ", less than the avg size, merging them");
-      return new MergeNormalizationPlan(candidateRegion.getFirst(),
-        candidateRegion2.getFirst());
+    if (plans.isEmpty()) {
+      LOG.debug("No normalization needed, regions look good for table: " + table);
+      return null;
     }
-    LOG.debug("No normalization needed, regions look good for table: " + table);
-    return EmptyNormalizationPlan.getInstance();
+    Collections.sort(plans, planComparator);
+    return plans;
   }
 
   private long getRegionSize(HRegionInfo hri) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/a5438f4a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index 2c302d5..9733d33 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -82,8 +82,8 @@ public class TestSimpleRegionNormalizer {
     Map<byte[], Integer> regionSizes = new HashMap<>();
 
     setupMocksForNormalizer(regionSizes, hris);
-    NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
-    assertTrue(plan instanceof EmptyNormalizationPlan);
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+    assertTrue(plans == null);
   }
 
   @Test
@@ -101,8 +101,8 @@ public class TestSimpleRegionNormalizer {
     regionSizes.put(hri2.getRegionName(), 15);
 
     setupMocksForNormalizer(regionSizes, hris);
-    NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
-    assertTrue((plan instanceof EmptyNormalizationPlan));
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+    assertTrue(plans == null);
   }
 
   @Test
@@ -128,8 +128,8 @@ public class TestSimpleRegionNormalizer {
     regionSizes.put(hri4.getRegionName(), 10);
 
     setupMocksForNormalizer(regionSizes, hris);
-    NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
-    assertTrue(plan instanceof EmptyNormalizationPlan);
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+    assertTrue(plans == null);
   }
 
   @Test
@@ -164,15 +164,16 @@ public class TestSimpleRegionNormalizer {
     regionSizes.put(hri5.getRegionName(), 16);
 
     setupMocksForNormalizer(regionSizes, hris);
-    NormalizationPlan plan = normalizer.computePlanForTable(testTable,
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable,
       mergeDesired ? bothTypes : splitType);
 
     if (mergeDesired) {
+      NormalizationPlan plan = plans.get(0);
       assertTrue(plan instanceof MergeNormalizationPlan);
       assertEquals(hri2, ((MergeNormalizationPlan) plan).getFirstRegion());
       assertEquals(hri3, ((MergeNormalizationPlan) plan).getSecondRegion());
     } else {
-      assertTrue(plan instanceof EmptyNormalizationPlan);
+      assertTrue(plans == null);
     }
   }
 
@@ -208,7 +209,8 @@ public class TestSimpleRegionNormalizer {
     regionSizes.put(hri6.getRegionName(), 2700);
 
     setupMocksForNormalizer(regionSizes, hris);
-    NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+    NormalizationPlan plan = plans.get(0);
 
     assertTrue(plan instanceof MergeNormalizationPlan);
     assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion());
@@ -242,9 +244,9 @@ public class TestSimpleRegionNormalizer {
     regionSizes.put(hri5.getRegionName(), 5);
 
     setupMocksForNormalizer(regionSizes, hris);
-    NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
 
-    assertTrue(plan instanceof EmptyNormalizationPlan);
+    assertTrue(plans == null);
   }
 
   @Test
@@ -270,7 +272,8 @@ public class TestSimpleRegionNormalizer {
     regionSizes.put(hri4.getRegionName(), 30);
 
     setupMocksForNormalizer(regionSizes, hris);
-    NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes);
+    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable, bothTypes);
+    NormalizationPlan plan = plans.get(0);
 
     assertTrue(plan instanceof SplitNormalizationPlan);
     assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo());

http://git-wip-us.apache.org/repos/asf/hbase/blob/a5438f4a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
index b821b8a..dc96dc7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
@@ -126,13 +126,19 @@ public class TestSimpleRegionNormalizerOnCluster {
     Thread.sleep(5000); // to let region load to update
     m.normalizeRegions();
 
-    while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6)
{
-      LOG.info("Waiting for normalization split to complete");
-      Thread.sleep(100);
+    while (true) {
+      List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
+      int cnt = 0;
+      for (HRegion region : regions) {
+        String regionName = region.getRegionInfo().getRegionNameAsString();
+        if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) {
+          cnt++;
+        }
+      }
+      if (cnt >= 2) {
+        break;
+      }
     }
-
-    assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
-
     admin.disableTable(TABLENAME);
     admin.deleteTable(TABLENAME);
   }


Mime
View raw message