hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From este...@apache.org
Subject hbase git commit: HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM
Date Fri, 11 Aug 2017 19:41:39 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 b181f172e -> 578e29f96


HBASE-18025 CatalogJanitor should collect outdated RegionStates from the AM


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/578e29f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/578e29f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/578e29f9

Branch: refs/heads/branch-1
Commit: 578e29f96b37875cd8092f7c6b3baf2b511148d0
Parents: b181f17
Author: Esteban Gutierrez <esteban@apache.org>
Authored: Fri Aug 11 12:56:20 2017 -0500
Committer: Esteban Gutierrez <esteban@apache.org>
Committed: Fri Aug 11 14:39:16 2017 -0500

----------------------------------------------------------------------
 .../hadoop/hbase/master/CatalogJanitor.java     |   4 +
 .../hadoop/hbase/master/RegionStates.java       |   6 +
 .../hadoop/hbase/master/ServerManager.java      |   7 +
 .../TestCatalogJanitorInMemoryStates.java       | 188 +++++++++++++++++++
 4 files changed, 205 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/578e29f9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
index 17644eb..00dc4a5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
@@ -217,6 +217,8 @@ public class CatalogJanitor extends ScheduledChore {
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
       MetaTableAccessor.deleteMergeQualifiers(services.getConnection(), mergedRegion);
+      services.getAssignmentManager().getRegionStates().deleteRegion(regionA);
+      services.getAssignmentManager().getRegionStates().deleteRegion(regionB);
       services.getServerManager().removeRegion(regionA);
       services.getServerManager().removeRegion(regionB);
       return true;
@@ -361,6 +363,8 @@ public class CatalogJanitor extends ScheduledChore {
       if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
       MetaTableAccessor.deleteRegion(this.connection, parent);
+      if (services.getAssignmentManager().getRegionStates() != null)
+        services.getAssignmentManager().getRegionStates().deleteRegion(parent);
       services.getServerManager().removeRegion(parent);
       result = true;
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/578e29f9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
index 082b5cc..599e649 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
@@ -899,6 +899,12 @@ public class RegionStates {
     }
   }
 
+  @VisibleForTesting
+  public boolean isRegionInRegionStates(final HRegionInfo hri) {
+    return (getRegionState(hri) != null || isRegionOnline(hri)) || isRegionInTransition(hri)
+        || isRegionInState(hri, State.OFFLINE, State.CLOSED);
+     }
+
   /**
    * Checking if a region was assigned to a server which is not online now.
    * If so, we should hold re-assign this region till SSH has split its wals.

http://git-wip-us.apache.org/repos/asf/hbase/blob/578e29f9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 93e532b..040342f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -1311,6 +1311,13 @@ public class ServerManager {
     flushedSequenceIdByRegion.remove(encodedName);
   }
 
+  @VisibleForTesting
+  public boolean isRegionInServerManagerStates(final HRegionInfo hri) {
+    final byte[] encodedName = hri.getEncodedNameAsBytes();
+    return (storeFlushedSequenceIdsByRegion.containsKey(encodedName)
+        || flushedSequenceIdByRegion.containsKey(encodedName));
+  }
+
   /**
    * Called by delete table and similar to notify the ServerManager that a region was removed.
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/578e29f9/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
new file mode 100644
index 0000000..d2bed9b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitorInMemoryStates.java
@@ -0,0 +1,188 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestCatalogJanitorInMemoryStates {
+  private static final Log LOG = LogFactory.getLog(TestCatalogJanitorInMemoryStates.class);
+  @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
+     withTimeout(this.getClass()).withLookingForStuckThread(true).build();
+  @Rule public final TestName name = new TestName();
+  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static byte [] ROW = Bytes.toBytes("testRow");
+  private static byte [] FAMILY = Bytes.toBytes("testFamily");
+  private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
+  private static byte [] VALUE = Bytes.toBytes("testValue");
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Test clearing a split parent from memory.
+   */
+  @Test(timeout = 180000)
+  public void testInMemoryParentCleanup() throws IOException, InterruptedException {
+    final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
+    final ServerManager sm = TEST_UTIL.getHBaseCluster().getMaster().getServerManager();
+    final CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().catalogJanitorChore;
+
+    Admin admin = TEST_UTIL.getHBaseAdmin();
+    admin.enableCatalogJanitor(false);
+
+    final TableName tableName = TableName.valueOf(name.getMethodName());
+    Table t = TEST_UTIL.createTable(tableName, FAMILY);
+    int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
+
+    RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
+    List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
+
+    // We need to create a valid split with daughter regions
+    HRegionLocation parent = allRegionLocations.get(0);
+    List<HRegionLocation> daughters = splitRegion(parent.getRegionInfo());
+    LOG.info("Parent region: " + parent);
+    LOG.info("Daughter regions: " + daughters);
+    assertNotNull("Should have found daughter regions for " + parent, daughters);
+
+    assertTrue("Parent region should exist in RegionStates",
+        am.getRegionStates().isRegionInRegionStates(parent.getRegionInfo()));
+    assertTrue("Parent region should exist in ServerManager",
+        sm.isRegionInServerManagerStates(parent.getRegionInfo()));
+
+    // clean the parent
+    Result r = MetaMockingUtil.getMetaTableRowResult(parent.getRegionInfo(), null,
+        daughters.get(0).getRegionInfo(), daughters.get(1).getRegionInfo());
+    // We need to wait a little before and after we remove the parent
+    Thread.sleep(5000);
+    janitor.cleanParent(parent.getRegionInfo(), r);
+    Thread.sleep(5000);
+    assertFalse("Parent region should have been removed from RegionStates",
+        am.getRegionStates().isRegionInRegionStates(parent.getRegionInfo()));
+    assertFalse("Parent region should have been removed from ServerManager",
+        sm.isRegionInServerManagerStates(parent.getRegionInfo()));
+
+  }
+
+  /*
+ * Splits a region
+ * @param t Region to split.
+ * @return List of region locations
+ * @throws IOException, InterruptedException
+ */
+  private List<HRegionLocation> splitRegion(final HRegionInfo r)
+      throws IOException, InterruptedException {
+    List<HRegionLocation> locations = new ArrayList<>();
+    // Split this table in two.
+    Admin admin = TEST_UTIL.getHBaseAdmin();
+    Connection connection = TEST_UTIL.getConnection();
+    admin.splitRegion(r.getEncodedNameAsBytes());
+    admin.close();
+    PairOfSameType<HRegionInfo> regions = waitOnDaughters(r);
+    if (regions != null) {
+      try (RegionLocator rl = connection.getRegionLocator(r.getTable())) {
+        locations.add(rl.getRegionLocation(regions.getFirst().getEncodedNameAsBytes()));
+        locations.add(rl.getRegionLocation(regions.getSecond().getEncodedNameAsBytes()));
+      }
+      return locations;
+    }
+    return locations;
+  }
+
+  /*
+   * Wait on region split. May return because we waited long enough on the split
+   * and it didn't happen.  Caller should check.
+   * @param r
+   * @return Daughter regions; caller needs to check table actually split.
+   */
+  private PairOfSameType<HRegionInfo> waitOnDaughters(final HRegionInfo r)
+      throws IOException {
+    long start = System.currentTimeMillis();
+    PairOfSameType<HRegionInfo> pair = null;
+    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
+         Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
+      Result result = null;
+      HRegionInfo region = null;
+      while ((System.currentTimeMillis() - start) < 60000) {
+        result = metaTable.get(new Get(r.getRegionName()));
+        if (result == null) {
+          break;
+        }
+        region = MetaTableAccessor.getHRegionInfo(result);
+        if (region.isSplitParent()) {
+          LOG.debug(region.toString() + " IS a parent!");
+          pair = MetaTableAccessor.getDaughterRegions(result);
+          break;
+        }
+        Threads.sleep(100);
+      }
+
+      if (pair.getFirst() == null || pair.getSecond() == null) {
+        throw new IOException("Failed to get daughters, for parent region: " + r);
+      }
+      return pair;
+    }
+  }
+}


Mime
View raw message