hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject [09/50] [abbrv] git commit: [master] Clear rootRegionLocation if the root is not served there
Date Thu, 31 Jul 2014 22:07:45 GMT
[master] Clear rootRegionLocation if the root is not served there

Summary: Clear `TableServers.rootRegionLocation` when receiving `NotServingRegionException`

Test Plan: `TestGetRegionLocation`

Reviewers: elliott, fan, manukranthk

Reviewed By: fan, manukranthk

Subscribers: hbase-eng@

Differential Revision: https://phabricator.fb.com/D1367998

Tasks: 4448200

git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42760 e7acf4d4-3532-417f-9e73-7a9ae25a1f51


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df1f91fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df1f91fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df1f91fa

Branch: refs/heads/0.89-fb
Commit: df1f91fad67a85315a80ff39beca640ea33ff4f4
Parents: 91eea3f
Author: daviddeng <daviddeng@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Mon Jun 9 19:02:34 2014 +0000
Committer: Elliott Clark <elliott@fb.com>
Committed: Thu Jul 31 14:44:22 2014 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/HRegionInfo.java    |  2 +-
 .../apache/hadoop/hbase/HTableDescriptor.java   |  4 +-
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |  4 +-
 .../hadoop/hbase/client/HTableMultiplexer.java  | 24 +++----
 .../hadoop/hbase/client/TableServers.java       | 20 +++---
 .../hadoop/hbase/HBaseTestingUtility.java       | 31 +++++++++
 .../regionserver/TestGetRegionLocation.java     | 69 +++++++++++++++++++-
 7 files changed, 126 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index dee47e0..9b30994 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -518,7 +518,7 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
     return this.tableDesc.isRootRegion();
   }
 
-  /** @return true if this is the meta table */
+  /** @return true if this is the META table */
   public boolean isMetaTable() {
     return this.tableDesc.isMetaTable();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 6f9b81a..5ddddfb 100644
--- a/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -35,7 +35,6 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 
-import com.google.common.collect.ImmutableSet;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -47,6 +46,7 @@ import org.apache.hadoop.io.WritableComparable;
 import com.facebook.swift.codec.ThriftConstructor;
 import com.facebook.swift.codec.ThriftField;
 import com.facebook.swift.codec.ThriftStruct;
+import com.google.common.collect.ImmutableSet;
 
 /**
  * HTableDescriptor contains the name of an HTable, and its
@@ -324,7 +324,7 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor>
{
     values.put(IS_META_KEY, isMeta? TRUE: FALSE);
   }
 
-  /** @return true if table is the meta table */
+  /** @return true if table is the META table */
   public boolean isMetaTable() {
     return isMetaRegion() && !isRootRegion();
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index ffeaf81..c16889b 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -91,7 +91,7 @@ public class HBaseAdmin {
     // use 30s pause time between retries instead of the small default
     // pause time. Shouldn't the RPC read timeout be also increased?
     this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, 30 * 1000);
-    this.numRetries = conf.getInt("hbase.client.retries.number", 5);
+    this.numRetries = conf.getInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);
     this.master = connection.getMaster();
   }
 
@@ -1441,7 +1441,7 @@ public class HBaseAdmin {
   public static void checkHBaseAvailable(Configuration conf)
   throws MasterNotRunningException {
     Configuration copyOfConf = HBaseConfiguration.create(conf);
-    copyOfConf.setInt("hbase.client.retries.number", 1);
+    copyOfConf.setInt(HConstants.CLIENT_RETRY_NUM_STRING, 1);
     new HBaseAdmin(copyOfConf);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
index a3360c0..dfee8c6 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
@@ -57,13 +57,13 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
  * Each put will be sharded into different buffer queues based on its destination region
server.
  * So each region server buffer queue will only have the puts which share the same destination.
  * And each queue will have a flush worker thread to flush the puts request to the region
server.
- * If any queue is full, the HTableMultiplexer starts to drop the Put requests for that 
+ * If any queue is full, the HTableMultiplexer starts to drop the Put requests for that
  * particular queue.
- * 
+ *
  * Also all the puts will be retried as a configuration number before dropping.
  * And the HTableMultiplexer can report the number of buffered requests and the number of
the
  * failed (dropped) requests in total or on per region server basis.
- * 
+ *
  * This class is thread safe.
  */
 public class HTableMultiplexer {
@@ -87,11 +87,11 @@ public class HTableMultiplexer {
   private final long frequency;
   //initial number of threads in the pool
   public static final int INITIAL_NUM_THREADS = 10;
-  
+
   /**
-   * 
+   *
    * @param conf The HBaseConfiguration
-   * @param perRegionServerBufferQueueSize determines the max number of the buffered Put
ops 
+   * @param perRegionServerBufferQueueSize determines the max number of the buffered Put
ops
    *         for each region server before dropping the request.
    */
   public HTableMultiplexer(Configuration conf, int perRegionServerBufferQueueSize) {
@@ -102,7 +102,7 @@ public class HTableMultiplexer {
     this.serverToFlushWorkerMap = new ConcurrentHashMap<HServerAddress, HTableFlushWorker>();
     this.tableNameToHTableMap = new ConcurrentSkipListMap<byte[], HTable>(
             Bytes.BYTES_COMPARATOR);
-    this.retryNum = conf.getInt("hbase.client.retries.number", 10);
+    this.retryNum = conf.getInt(HConstants.CLIENT_RETRY_NUM_STRING, 10);
     this.retriedInQueueMax = conf.getInt("hbase.client.retried.inQueue", 10000);
     this.perRegionServerBufferQueueSize = perRegionServerBufferQueueSize;
     this.frequency = conf.getLong("hbase.htablemultiplexer.flush.frequency.ms",
@@ -127,7 +127,7 @@ public class HTableMultiplexer {
   }
 
   /**
-   * The puts request will be buffered by their corresponding buffer queue. 
+   * The puts request will be buffered by their corresponding buffer queue.
    * Return the list of puts which could not be queued.
    * @param table
    * @param put
@@ -138,13 +138,13 @@ public class HTableMultiplexer {
       HBaseRPCOptions options) throws IOException {
     if (puts == null)
       return null;
-    
+
     List <Put> failedPuts = null;
     boolean result;
     for (Put put : puts) {
       result = put(table, put, this.retryNum, options);
       if (result == false) {
-        
+
         // Create the failed puts list if necessary
         if (failedPuts == null) {
           failedPuts = new ArrayList<Put>();
@@ -183,7 +183,7 @@ public class HTableMultiplexer {
         LinkedBlockingQueue<PutStatus> queue = getBufferedQueue(addr);
         // Generate a MultiPutStatus obj and offer it into the queue
         PutStatus s = new PutStatus(loc.getRegionInfo(), put, retry, options);
-        
+
         return queue.offer(s);
       }
     } catch (Exception e) {
@@ -480,7 +480,7 @@ public class HTableMultiplexer {
       return this.tableNameToHTableMap.size();
     }
   }
-  
+
   private static class PutStatus {
     private final HRegionInfo regionInfo;
     private final Put put;

http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
index bc38cd6..bb671fb 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/TableServers.java
@@ -51,11 +51,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListenableFutureTask;
-import com.google.common.util.concurrent.ListeningExecutorService;
+import javax.annotation.Nullable;
+
 import org.apache.commons.lang.mutable.MutableBoolean;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -97,8 +94,11 @@ import org.apache.hadoop.ipc.RemoteException;
 import org.apache.thrift.transport.TTransportException;
 
 import com.google.common.base.Preconditions;
-
-import javax.annotation.Nullable;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ListeningExecutorService;
 
 /* Encapsulates finding the servers for an HBase instance */
 public class TableServers implements ServerConnection {
@@ -1079,8 +1079,6 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
             parentTable, row);
           metaCache.add(tableName, location);
         }
-      } catch (Throwable t) {
-        throw t;
       } finally {
         HBaseThriftRPC.isMeta.get().pop();
       }
@@ -1094,6 +1092,10 @@ private HRegionLocation locateMetaInRoot(final byte[] row,
       // already processed this. Don't process this again.
       throw e;
     } catch (IOException e) {
+        if (e instanceof NotServingRegionException) {
+          this.rootRegionLocation = null;
+        }
+
       if (e instanceof RemoteException) {
         e = RemoteExceptionHandler
           .decodeRemoteException((RemoteException) e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 101e552..10762fb 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -1561,6 +1561,37 @@ public class HBaseTestingUtility {
     return list;
   }
 
+  /**
+   * @return the only HRegion for ROOT table. Returns a null if not found.
+   *         It loops over all regions to find it, could be slow.
+   */
+  public HRegion getRootRegion() {
+    for (HRegionServer rs : getOnlineRegionServers()) {
+      for (HRegion region : rs.getOnlineRegions()) {
+        if (region.getRegionInfo().isRootRegion()) {
+          return region;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * @return one of HRegion for META table. Returns a null if not found.
+   *         It loops over all regions to find it, could be slow.
+   */
+  public HRegion getMetaRegion() {
+    for (HRegionServer rs : getOnlineRegionServers()) {
+      for (HRegion region : rs.getOnlineRegions()) {
+        if (region.getRegionInfo().isMetaTable()) {
+          return region;
+        }
+      }
+    }
+
+    return null;
+  }
 
   /**
    * Wait until <code>countOfRegion</code> in .META. have a non-empty

http://git-wip-us.apache.org/repos/asf/hbase/blob/df1f91fa/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
index efb24b4..de391c5 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetRegionLocation.java
@@ -30,13 +30,17 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HServerInfo;
 import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
 import org.apache.hadoop.hbase.client.TableServers;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.StringBytes;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -48,7 +52,7 @@ public class TestGetRegionLocation {
   private static final Log LOG = LogFactory.getLog(TestGetRegionLocation.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private final static Configuration CONF = TEST_UTIL.getConfiguration();
-  private final static byte[] TABLE = Bytes.toBytes("table");
+  private final static StringBytes TABLE = new StringBytes("table");
   private final static byte[] FAMILY = Bytes.toBytes("family");
   private final static byte[][] FAMILIES = { FAMILY };
   private final static byte[] START_KEY = Bytes.toBytes("aaa");
@@ -58,10 +62,13 @@ public class TestGetRegionLocation {
 
   @Before
   public void setUp() throws IOException, InterruptedException {
+    TEST_UTIL.getConfiguration().setInt(HConstants.CLIENT_RETRY_NUM_STRING, 5);
+
     // Use assignment plan so that regions are not moved unexpectedly.
     TEST_UTIL.useAssignmentLoadBalancer();
     TEST_UTIL.startMiniCluster(NUM_SLAVES);
-    TEST_UTIL.createTable(TABLE, FAMILIES, 1, START_KEY, END_KEY, NUM_REGIONS);
+    TEST_UTIL.createTable(TABLE, FAMILIES, 1, START_KEY, END_KEY, NUM_REGIONS)
+        .close();
   }
 
   @After
@@ -69,6 +76,64 @@ public class TestGetRegionLocation {
     TEST_UTIL.shutdownMiniCluster();
   }
 
+  @Test(timeout = 150000)
+  public void testMetaRootRegionRelocation() throws Exception {
+    Assert.assertTrue("We should have more than one region servers",
+        NUM_SLAVES >= 3);
+
+    try (TableServers connection = new TableServers(CONF)) {
+      HRegionLocation location =
+          connection.getRegionLocation(TABLE, HConstants.EMPTY_START_ROW, true);
+      Assert.assertNotNull("Cannot get location", location);
+
+      HRegion rootRegion = TEST_UTIL.getRootRegion();
+      Assert.assertNotNull("Cannot find ROOT region", rootRegion);
+      HRegionServer rootServer =
+          TEST_UTIL.getRSWithRegion(rootRegion.getRegionName());
+
+      HRegion metaRegion = TEST_UTIL.getMetaRegion();
+      Assert.assertNotNull("Cannot find META region", metaRegion);
+      HRegionServer metaServer =
+          TEST_UTIL.getRSWithRegion(metaRegion.getRegionName());
+
+      TEST_UTIL.getMiniHBaseCluster().getMaster()
+          .addServerToBlacklist(rootServer.getServerInfo().getHostnamePort());
+      TEST_UTIL.getMiniHBaseCluster().getMaster()
+          .addServerToBlacklist(metaServer.getServerInfo().getHostnamePort());
+
+      rootServer.closeRegion(rootRegion.getRegionInfo(), true);
+
+      metaServer.closeRegion(metaRegion.getRegionInfo(), true);
+
+      try {
+        rootServer.getRegion(rootRegion.getRegionName());
+        Assert.fail("Should throw NotServingRegionException");
+      } catch (NotServingRegionException e) {
+        // good
+      }
+      try {
+        metaServer.getRegion(metaRegion.getRegionName());
+        Assert.fail("Should throw NotServingRegionException");
+      } catch (NotServingRegionException e) {
+        // good
+      }
+
+      try {
+        connection.listTables();
+      } catch (RetriesExhaustedException e) {
+        e.printStackTrace();
+        Assert.fail("listTable failed: " + e);
+      }
+
+      try {
+        connection.listTables();
+      } catch (NotServingRegionException e) {
+        e.printStackTrace();
+        Assert.fail("listTable failed: " + e);
+      }
+    }
+  }
+
   /**
    * This test tests the getRegionLocation() call in the
    * (Thrift)HRegionInterface API.


Mime
View raw message