hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1567266 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/client/ test/java/org/apache/hadoop/hbase/client/
Date Tue, 11 Feb 2014 18:28:28 GMT
Author: liyin
Date: Tue Feb 11 18:28:28 2014
New Revision: 1567266

URL: http://svn.apache.org/r1567266
Log:
[HBASE-10360] Auto initialized all the HRegionLocations even though the forceFetch is false.

Author: liyintang

Summary:
The HTable.getCachedHRegionLocations(false) only returns all the cached HRegionLocations based
on the original design. Per discussions with Calypso team, we decided to enhance the API so
that even though the forceFetch is set as false, if the HRegionLocations for that table has
not been initialized yet, it will fetch all the HRegionLocations from the META table.

This enhancement has been implemented in HConnectionManager instead of HTable layer. Therefore,
multiple HTable instances for the same table could share this "initialized" state.

Test Plan: TestHTable

Reviewers: manukranthk

Reviewed By: manukranthk

CC: hbase-dev@

Differential Revision: https://phabricator.fb.com/D1157181

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTable.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=1567266&r1=1567265&r2=1567266&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Tue
Feb 11 18:28:28 2014
@@ -19,6 +19,12 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -27,17 +33,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
-
 /**
  * Cluster connection.
  * {@link HConnectionManager} manages instances of this class.
@@ -226,7 +221,7 @@ public interface HConnection extends Clo
   /**
    * Process a batch of Gets. Does the retries.
    *
-   * @param list
+   * @param actions
    *          A batch of Gets to process.
    * @param tableName
    *          The name of the table
@@ -425,18 +420,12 @@ public interface HConnection extends Clo
   public String getServerConfProperty(String prop) throws IOException;
 
   /**
-   * Initialize all the HRegionLocations by scanning the META table and cache these locations
-   * properly.
-   * @param conf
-   * @param tableName
-   */
-  public void prefetchHRegionLocations(final Configuration conf, final byte[] tableName);
-
-
-  /**
-   * Get all the cached HRegionLocations
-   * @param tableName
+   * Get all the cached HRegionLocations for the given table. If the given table has not
been
+   * initialized for all HRegionLocations, force to refresh the locations from META.
+   * @param tableName The given tableName
+   * @param forceRefresh Whether to force to refresh the HRegionLocations from META.
    * @return cachedHRegionLocations
    */
-  public Collection<HRegionLocation> getCachedHRegionLocations(final byte [] tableName);
+  public Collection<HRegionLocation> getCachedHRegionLocations(final byte [] tableName,
+                                                               boolean forceRefresh);
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1567266&r1=1567265&r2=1567266&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Tue Feb 11 18:28:28 2014
@@ -19,70 +19,16 @@
  */
 package org.apache.hadoop.hbase.client;
 
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.SyncFailedException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.net.ConnectException;
-import java.net.SocketTimeoutException;
-import java.nio.channels.ClosedChannelException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.CopyOnWriteArraySet;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
 import org.apache.commons.lang.mutable.MutableBoolean;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.NotServingRegionException;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
-import org.apache.hadoop.hbase.ipc.HBaseRPC;
-import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
-import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.ipc.*;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RegionOverloadedException;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.MetaUtils;
-import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.util.*;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
@@ -94,6 +40,20 @@ import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.Watcher.Event.EventType;
 import org.apache.zookeeper.Watcher.Event.KeeperState;
 
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.SyncFailedException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.nio.channels.ClosedChannelException;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
 /**
  * A non-instantiable class that manages connections to multiple tables in
  * multiple HBase instances.
@@ -517,6 +477,9 @@ public class HConnectionManager {
     Map<String, ConcurrentMap<HRegionInfo, HRegionLocation>> batchedUploadUpdatesMap;
     private int batchedUploadSoftFlushRetries;
     private long batchedUploadSoftFlushTimeoutMillis;
+
+    private ConcurrentSkipListSet<byte[]> initializedTableSet =
+      new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
     /**
      * constructor
      * @param conf Configuration object
@@ -1184,11 +1147,13 @@ public class HConnectionManager {
               e instanceof EOFException);
     }
 
-    public void prefetchHRegionLocations(final Configuration conf, final byte[] tableName)
{
-      prefetchRegionCache(tableName, null, Integer.MAX_VALUE);
-    }
+    public Collection<HRegionLocation> getCachedHRegionLocations(final byte [] tableName,
+                                                                 boolean forceRefresh) {
+      if (forceRefresh || !initializedTableSet.contains(tableName)) {
+        prefetchRegionCache(tableName, null, Integer.MAX_VALUE);
+        initializedTableSet.add(tableName);
+      }
 
-    public Collection<HRegionLocation> getCachedHRegionLocations(final byte [] tableName)
{
       ConcurrentSkipListMap<byte [], HRegionLocation> tableLocations =
         getTableLocations(tableName);
       return tableLocations.values();

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1567266&r1=1567265&r2=1567266&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java Tue Feb
11 18:28:28 2014
@@ -468,11 +468,7 @@ public class HTable implements HTableInt
    */
   @Override
   public Collection<HRegionLocation> getCachedHRegionLocations(boolean forceRefresh)
{
-    if (forceRefresh) {
-      connection.prefetchHRegionLocations(configuration, tableName);
-    }
-
-    return connection.getCachedHRegionLocations(tableName);
+    return connection.getCachedHRegionLocations(tableName, forceRefresh);
   }
 
 

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTable.java?rev=1567266&r1=1567265&r2=1567266&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTable.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTable.java Tue
Feb 11 18:28:28 2014
@@ -50,7 +50,9 @@ public class TestHTable {
       3, Bytes.toBytes("aaaaa"), Bytes.toBytes("zzzzz"), NUM_REGIONS);
 
     NavigableMap<HRegionInfo, HServerAddress> allRegionsInfoMap =  table.getRegionsInfo();
-    Collection<HRegionLocation> regionLocations = table.getCachedHRegionLocations(true);
+    Collection<HRegionLocation> regionLocations = table.getCachedHRegionLocations(false);
+    Collection<HRegionLocation> regionLocations2 = table.getCachedHRegionLocations(true);
+    assertEquals(regionLocations.size(), regionLocations2.size());
 
     // Make sure allRegions and regionLocations containing the same information
     verifyOnlineRegionsAndRegionLocations(allRegionsInfoMap, regionLocations, NUM_REGIONS);



Mime
View raw message