From commits-return-93754-archive-asf-public=cust-asf.ponee.io@hbase.apache.org Tue May 26 10:44:53 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 89031180608 for ; Tue, 26 May 2020 12:44:53 +0200 (CEST) Received: (qmail 29201 invoked by uid 500); 26 May 2020 10:44:52 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 29192 invoked by uid 99); 26 May 2020 10:44:52 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 May 2020 10:44:52 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 07C46819F5; Tue, 26 May 2020 10:44:51 +0000 (UTC) Date: Tue, 26 May 2020 10:44:51 +0000 To: "commits@hbase.apache.org" Subject: [hbase] branch branch-2 updated: HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <159048989082.26970.529604406671656480@gitbox.apache.org> From: zghao@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hbase X-Git-Refname: refs/heads/branch-2 X-Git-Reftype: branch X-Git-Oldrev: 51a84025cb2ea89cdef5472be576cfe8aadce8e1 X-Git-Newrev: d285c6b681d602d67daca3a365fe88546ef32b8c X-Git-Rev: d285c6b681d602d67daca3a365fe88546ef32b8c X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. zghao pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hbase.git The following commit(s) were added to refs/heads/branch-2 by this push: new d285c6b HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770) d285c6b is described below commit d285c6b681d602d67daca3a365fe88546ef32b8c Author: XinSun AuthorDate: Tue May 26 17:03:14 2020 +0800 HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770) Signed-off-by: Guanghao Zhang --- .../normalizer/AbstractRegionNormalizer.java | 10 +++++ .../hbase/master/normalizer/MergeNormalizer.java | 10 ++--- .../master/normalizer/SimpleRegionNormalizer.java | 25 ++++++------- .../normalizer/TestSimpleRegionNormalizer.java | 43 ++++++++++++++++++++++ 4 files changed, 69 insertions(+), 19 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java index 8b17f77..c79c1d4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/AbstractRegionNormalizer.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Objects; + import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Size; @@ -42,8 +43,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; @InterfaceAudience.Private public abstract class AbstractRegionNormalizer implements RegionNormalizer { private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class); + + public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY = + "hbase.normalizer.min.region.count"; + public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3; + protected MasterServices masterServices; protected MasterRpcServices masterRpcServices; + protected int minRegionCount; /** * Set the master service. @@ -52,6 +59,9 @@ public abstract class AbstractRegionNormalizer implements RegionNormalizer { @Override public void setMasterServices(MasterServices masterServices) { this.masterServices = masterServices; + minRegionCount = masterServices.getConfiguration().getInt( + HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, + HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java index 444c27c..f006103 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizer.java @@ -23,10 +23,10 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,13 +63,13 @@ import org.slf4j.LoggerFactory; public class MergeNormalizer extends AbstractRegionNormalizer { private static final Logger LOG = LoggerFactory.getLogger(MergeNormalizer.class); - private int minRegionCount; private int minRegionAge; private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; - public MergeNormalizer() { - Configuration conf = HBaseConfiguration.create(); - minRegionCount = conf.getInt("hbase.normalizer.min.region.count", 3); + @Override + public void setMasterServices(MasterServices masterServices) { + super.setMasterServices(masterServices); + Configuration conf = masterServices.getConfiguration(); minRegionAge = conf.getInt("hbase.normalizer.min.region.merge.age", 3); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index bd90f5b..b68c068 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import org.apache.hadoop.hbase.HBaseConfiguration; + import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -51,13 +51,8 @@ import org.slf4j.LoggerFactory; public class SimpleRegionNormalizer extends AbstractRegionNormalizer { private static final Logger LOG = LoggerFactory.getLogger(SimpleRegionNormalizer.class); - private int minRegionCount; private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; - public SimpleRegionNormalizer() { - minRegionCount = HBaseConfiguration.create().getInt("hbase.normalizer.min.region.count", 3); - } - @Override public void planSkipped(RegionInfo hri, PlanType type) { skippedCount[type.ordinal()]++; @@ -112,11 +107,7 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer { List tableRegions = masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table); - if (tableRegions == null || tableRegions.size() < minRegionCount) { - int nrRegions = tableRegions == null ? 0 : tableRegions.size(); - LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is " - + "{}, not running normalizer", - table, nrRegions, minRegionCount); + if (tableRegions == null) { return null; } @@ -131,9 +122,15 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer { } if (mergeEnabled) { - List mergePlans = getMergeNormalizationPlan(table); - if (mergePlans != null) { - plans.addAll(mergePlans); + if (tableRegions.size() < minRegionCount) { + LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run" + + " is {}, not running normalizer", + table, tableRegions.size(), minRegionCount); + } else { + List mergePlans = getMergeNormalizationPlan(table); + if (mergePlans != null) { + plans.addAll(mergePlans); + } } } if (plans.isEmpty()) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index a05b3a8..5c1c4e0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.normalizer; import static java.lang.String.format; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -30,7 +31,10 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ServerName; @@ -526,6 +530,45 @@ public class TestSimpleRegionNormalizer { assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion()); } + @Test + public void testSplitIfTooFewRegions() throws HBaseIOException { + final TableName tableName = TableName.valueOf(name.getMethodName()); + List RegionInfo = new ArrayList<>(); + Map regionSizes = new HashMap<>(); + + RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("aaa")) + .setEndKey(Bytes.toBytes("bbb")) + .build(); + RegionInfo.add(hri1); + regionSizes.put(hri1.getRegionName(), 1); + + RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("bbb")) + .setEndKey(Bytes.toBytes("ccc")) + .build(); + RegionInfo.add(hri2); + regionSizes.put(hri2.getRegionName(), 1); + // the third region is huge one + RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName) + .setStartKey(Bytes.toBytes("ccc")) + .setEndKey(Bytes.toBytes("ddd")) + .build(); + RegionInfo.add(hri3); + regionSizes.put(hri3.getRegionName(), 10); + + setupMocksForNormalizer(regionSizes, RegionInfo); + + Configuration configuration = HBaseConfiguration.create(); + configuration.setInt(AbstractRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4); + when(masterServices.getConfiguration()).thenReturn(configuration); + + List plans = normalizer.computePlanForTable(tableName); + assertNotNull(plans); + NormalizationPlan plan = plans.get(0); + assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo()); + } + @SuppressWarnings("MockitoCast") private void setupMocksForNormalizer(Map regionSizes, List RegionInfo) {