hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [4/5] hbase git commit: HBASE-19364, Truncate_preserve fails with table when replica region > 1
Date Sat, 24 Feb 2018 01:55:27 GMT
HBASE-19364, Truncate_preserve fails with table when replica region > 1


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d8fee21
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d8fee21
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d8fee21

Branch: refs/heads/branch-1.4
Commit: 0d8fee2158e08bc6d0907d4abbe1215eaded6ce3
Parents: 1b0b8d9
Author: Pankaj Kumar <pankaj.kr@huawei.com>
Authored: Thu Dec 7 22:51:01 2017 +0530
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Fri Feb 23 17:03:03 2018 -0800

----------------------------------------------------------------------
 .../apache/hadoop/hbase/MetaTableAccessor.java  | 45 +++++++++++++++++---
 .../master/procedure/ProcedureSyncWait.java     | 24 ++++++-----
 .../procedure/TruncateTableProcedure.java       |  2 +-
 .../MasterProcedureTestingUtility.java          |  5 +++
 .../procedure/TestTruncateTableProcedure.java   | 44 +++++++++++++------
 5 files changed, 90 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 3f11558..440f8c6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -392,12 +392,27 @@ public class MetaTableAccessor {
    * @return Ordered list of {@link HRegionInfo}.
    * @throws IOException
    */
-  public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw,
-      Connection connection, TableName tableName, final boolean excludeOfflinedSplitParents)
-        throws IOException {
+  public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection
connection,
+      TableName tableName, final boolean excludeOfflinedSplitParents) throws IOException
{
+    return getTableRegions(zkw, connection, tableName, excludeOfflinedSplitParents, false);
+  }
+
+  /**
+   * Gets all of the regions of the specified table.
+   * @param zkw zookeeper connection to access meta table
+   * @param connection connection we're using
+   * @param tableName table we're looking for
+   * @param excludeOfflinedSplitParents If true, do not include offlined split parents in
the
+   *          return.
+   * @param excludeReplicaRegions If true, do not include replica regions in the result.
+   * @return Ordered list of {@link HRegionInfo}.
+   */
+  public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection
connection,
+      TableName tableName, final boolean excludeOfflinedSplitParents,
+      final boolean excludeReplicaRegions) throws IOException {
     List<Pair<HRegionInfo, ServerName>> result = null;
-      result = getTableRegionsAndLocations(zkw, connection, tableName,
-        excludeOfflinedSplitParents);
+    result = getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
+      excludeReplicaRegions);
     return getListOfHRegionInfos(result);
   }
 
@@ -478,7 +493,22 @@ public class MetaTableAccessor {
   public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
       ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
       final boolean excludeOfflinedSplitParents) throws IOException {
+    return getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
+      false);
+  }
 
+  /**
+   * @param zkw ZooKeeperWatcher instance we're using to get hbase:meta location
+   * @param connection connection we're using
+   * @param tableName table to work with
+   * @param excludeOfflinedSplitParents Exclude offline regions
+   * @param excludeReplicaRegions If true, do not include replica regions in the result.
+   * @return List of regioninfos and server addresses.
+   */
+  public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
+      ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
+      final boolean excludeOfflinedSplitParents, final boolean excludeReplicaRegions)
+          throws IOException {
     if (tableName.equals(TableName.META_TABLE_NAME)) {
       // If meta, do a bit of special handling.
       ServerName serverName = new MetaTableLocator().getMetaRegionLocation(zkw);
@@ -514,6 +544,11 @@ public class MetaTableAccessor {
           }
           for (HRegionLocation loc : current.getRegionLocations()) {
             if (loc != null) {
+              // Exclude replica region from the result
+              if (excludeReplicaRegions
+                  && !RegionReplicaUtil.isDefaultReplica(loc.getRegionInfo())) {
+                continue;
+              }
               this.results.add(new Pair<HRegionInfo, ServerName>(
                 loc.getRegionInfo(), loc.getServerName()));
             }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
index d0dea0a..1846cef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
@@ -149,22 +149,24 @@ public final class ProcedureSyncWait {
 
   protected static List<HRegionInfo> getRegionsFromMeta(final MasterProcedureEnv env,
       final TableName tableName) throws IOException {
-    return getRegionsFromMeta(env, tableName, false);
+    return getRegionsFromMeta(env, tableName, false, false);
   }
 
   protected static List<HRegionInfo> getRegionsFromMeta(final MasterProcedureEnv env,
-      final TableName tableName,  final boolean excludeOfflinedSplitParents) throws IOException
{
+      final TableName tableName, final boolean excludeOfflinedSplitParents,
+      final boolean excludeReplicaRegions) throws IOException {
     return ProcedureSyncWait.waitFor(env, "regions of table=" + tableName + " from meta",
-        new ProcedureSyncWait.Predicate<List<HRegionInfo>>() {
-          @Override
-          public List<HRegionInfo> evaluate() throws IOException {
-            if (TableName.META_TABLE_NAME.equals(tableName)) {
-              return new MetaTableLocator().getMetaRegions(env.getMasterServices().getZooKeeper());
-            }
-            return MetaTableAccessor.getTableRegions(env.getMasterServices().getZooKeeper(),
-                env.getMasterServices().getConnection(), tableName, excludeOfflinedSplitParents);
+      new ProcedureSyncWait.Predicate<List<HRegionInfo>>() {
+        @Override
+        public List<HRegionInfo> evaluate() throws IOException {
+          if (TableName.META_TABLE_NAME.equals(tableName)) {
+            return new MetaTableLocator().getMetaRegions(env.getMasterServices().getZooKeeper());
           }
-        });
+          return MetaTableAccessor.getTableRegions(env.getMasterServices().getZooKeeper(),
+            env.getMasterServices().getConnection(), tableName, excludeOfflinedSplitParents,
+            excludeReplicaRegions);
+        }
+      });
   }
 
   protected static void waitRegionInTransition(final MasterProcedureEnv env,

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 735e94f..120ae4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -85,7 +85,7 @@ public class TruncateTableProcedure
 
           // TODO: Move out... in the acquireLock()
           LOG.debug("waiting for '" + getTableName() + "' regions in transition");
-          regions = ProcedureSyncWait.getRegionsFromMeta(env, getTableName(), true);
+          regions = ProcedureSyncWait.getRegionsFromMeta(env, getTableName(), true, true);
           assert regions != null && !regions.isEmpty() : "unexpected 0 regions";
           ProcedureSyncWait.waitRegionInTransition(env, regions);
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index fa57b12..ff479d4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -74,6 +74,11 @@ public class MasterProcedureTestingUtility {
   public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv>
procExec,
       final TableName tableName, final byte[][] splitKeys, String... family) throws IOException
{
     HTableDescriptor htd = createHTD(tableName, family);
+    return createTable(procExec, htd, splitKeys);
+  }
+
+  public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv>
procExec,
+      HTableDescriptor htd, final byte[][] splitKeys) throws IOException {
     HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
     long procId = ProcedureTestingUtility.submitAndWait(procExec,
       new CreateTableProcedure(procExec.getEnvironment(), htd, regions));

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8fee21/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 8797a4f..47460af 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -43,6 +43,8 @@ import org.junit.experimental.categories.Category;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
+
 @Category(MediumTests.class)
 public class TestTruncateTableProcedure {
   private static final Log LOG = LogFactory.getLog(TestTruncateTableProcedure.class);
@@ -241,23 +243,40 @@ public class TestTruncateTableProcedure {
     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
   }
 
-  @Test(timeout=60000)
-  public void testTruncateWithPreserveAfterSplit() throws Exception{
+  @Test(timeout = 60000)
+  public void testTruncateWithPreserveAfterSplit() throws Exception {
     final String[] families = new String[] { "f1", "f2" };
-    final byte[][] splitKeys = new byte[][] {
-        Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
-    };
+    final byte[][] splitKeys =
+        new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
     TableName tableName = TableName.valueOf("testTruncateWithPreserveAfterSplit");
-    HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
-        getMasterProcedureExecutor(), tableName, splitKeys, families);
+    HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
+      tableName, splitKeys, families);
+    splitAndTruncate(families, splitKeys, tableName, regions);
+  }
+
+  @Test(timeout = 60000)
+  public void testTruncatePreserveWithReplicaRegionAfterSplit() throws Exception {
+    final String[] families = new String[] { "f1", "f2" };
+    final byte[][] splitKeys =
+        new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+    TableName tableName = TableName.valueOf("testTruncateWithPreserveAfterSplit");
+    HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, families);
+    htd.setRegionReplication(3);
+    HRegionInfo[] regions =
+        MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), htd, splitKeys);
+    splitAndTruncate(families, splitKeys, tableName, regions);
+  }
+
+  private void splitAndTruncate(final String[] families, final byte[][] splitKeys,
+      TableName tableName, HRegionInfo[] regions) throws IOException, InterruptedException
{
     // load enough data so the table can split
-    MasterProcedureTestingUtility.loadData(
-        UTIL.getConnection(), tableName, 5000, splitKeys, families);
+    MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 5000, splitKeys,
+      families);
     assertEquals(5000, UTIL.countRows(tableName));
     UTIL.getHBaseAdmin().split(tableName);
     UTIL.waitUntilAllRegionsAssigned(tableName);
-    //wait until split really happens
-    while(UTIL.getHBaseAdmin().getTableRegions(tableName).size() <= regions.length) {
+    // wait until split really happens
+    while (UTIL.getHBaseAdmin().getTableRegions(tableName).size() <= regions.length) {
       Thread.sleep(50);
     }
     // disable the table
@@ -265,10 +284,9 @@ public class TestTruncateTableProcedure {
     // truncate the table
     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
     long procId = ProcedureTestingUtility.submitAndWait(procExec,
-        new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));
+      new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));
     ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
 
     UTIL.waitUntilAllRegionsAssigned(tableName);
-
   }
 }


Mime
View raw message