hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-18035 Meta replica does not give any primaryOperationTimeout to primary meta region (huaxiang sun)
Date Sun, 21 May 2017 13:39:40 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 300c5388f -> c03f003f4


HBASE-18035 Meta replica does not give any primaryOperationTimeout to primary meta region
(huaxiang sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c03f003f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c03f003f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c03f003f

Branch: refs/heads/branch-1
Commit: c03f003f4f0301091fdcb1753691bcf7d46bd38c
Parents: 300c538
Author: tedyu <yuzhihong@gmail.com>
Authored: Sun May 21 06:39:33 2017 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Sun May 21 06:39:33 2017 -0700

----------------------------------------------------------------------
 .../hbase/client/ConnectionConfiguration.java   | 18 +++--
 .../hadoop/hbase/client/ConnectionManager.java  | 17 ++++-
 .../org/apache/hadoop/hbase/HConstants.java     |  7 ++
 .../hbase/client/TestReplicaWithCluster.java    | 76 ++++++++++++++++++++
 4 files changed, 112 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/c03f003f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionConfiguration.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionConfiguration.java
index de760d4..76dcdfa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionConfiguration.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionConfiguration.java
@@ -40,6 +40,7 @@ public class ConnectionConfiguration {
   private final long scannerMaxResultSize;
   private final int primaryCallTimeoutMicroSecond;
   private final int replicaCallTimeoutMicroSecondScan;
+  private final int metaReplicaCallTimeoutMicroSecondScan;
   private final int retries;
   private final int maxKeyValueSize;
 
@@ -50,9 +51,8 @@ public class ConnectionConfiguration {
   ConnectionConfiguration(Configuration conf) {
     this.writeBufferSize = conf.getLong(WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT);
 
-    this.metaOperationTimeout = conf.getInt(
-      HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT,
-      HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
+    this.metaOperationTimeout = conf.getInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT,
+        HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
 
     this.operationTimeout = conf.getInt(
       HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
@@ -62,7 +62,7 @@ public class ConnectionConfiguration {
 
     this.scannerMaxResultSize =
         conf.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
-          HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
+            HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
 
     this.primaryCallTimeoutMicroSecond =
         conf.getInt("hbase.client.primaryCallTimeout.get", 10000); // 10ms
@@ -70,6 +70,10 @@ public class ConnectionConfiguration {
     this.replicaCallTimeoutMicroSecondScan =
         conf.getInt("hbase.client.replicaCallTimeout.scan", 1000000); // 1000 ms
 
+    this.metaReplicaCallTimeoutMicroSecondScan =
+        conf.getInt(HConstants.HBASE_CLIENT_MEAT_REPLICA_SCAN_TIMEOUT,
+            HConstants.HBASE_CLIENT_MEAT_REPLICA_SCAN_TIMEOUT_DEFAULT);
+
     this.retries = conf.getInt(
        HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
 
@@ -90,6 +94,8 @@ public class ConnectionConfiguration {
     this.scannerMaxResultSize = HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE;
     this.primaryCallTimeoutMicroSecond = 10000;
     this.replicaCallTimeoutMicroSecondScan = 1000000;
+    this.metaReplicaCallTimeoutMicroSecondScan =
+        HConstants.HBASE_CLIENT_MEAT_REPLICA_SCAN_TIMEOUT_DEFAULT;
     this.retries = HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER;
     this.maxKeyValueSize = MAX_KEYVALUE_SIZE_DEFAULT;
   }
@@ -118,6 +124,10 @@ public class ConnectionConfiguration {
     return replicaCallTimeoutMicroSecondScan;
   }
 
+  public int getMetaReplicaCallTimeoutMicroSecondScan() {
+    return metaReplicaCallTimeoutMicroSecondScan;
+  }
+
   public int getRetriesNumber() {
     return retries;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c03f003f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 60d4217..aa44070 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -565,7 +565,8 @@ class ConnectionManager {
     private final boolean hostnamesCanChange;
     private final long pause;
     private final long pauseForCQTBE;// pause for CallQueueTooBigException, if specified
-    private final boolean useMetaReplicas;
+    private boolean useMetaReplicas;
+    private final int metaReplicaCallTimeoutScanInMicroSecond;
     private final int numTries;
     final int rpcTimeout;
     final int writeRpcTimeout;
@@ -668,6 +669,9 @@ class ConnectionManager {
       }
       this.useMetaReplicas = conf.getBoolean(HConstants.USE_META_REPLICAS,
           HConstants.DEFAULT_USE_META_REPLICAS);
+      this.metaReplicaCallTimeoutScanInMicroSecond =
+          connectionConfig.getMetaReplicaCallTimeoutMicroSecondScan();
+
       this.numTries = connectionConfig.getRetriesNumber();
       this.rpcTimeout = conf.getInt(
           HConstants.HBASE_RPC_TIMEOUT_KEY,
@@ -737,6 +741,14 @@ class ConnectionManager {
       }
     }
 
+    /**
+     * @param useMetaReplicas
+     */
+    @VisibleForTesting
+    void setUseMetaReplicas(final boolean useMetaReplicas) {
+      this.useMetaReplicas = useMetaReplicas;
+    }
+
     @Override
     public HTableInterface getTable(String tableName) throws IOException {
       return getTable(TableName.valueOf(tableName));
@@ -1294,7 +1306,8 @@ class ConnectionManager {
           s.resetMvccReadPoint();
           try (ReversedClientScanner rcs =
               new ReversedClientScanner(conf, s, TableName.META_TABLE_NAME, this, rpcCallerFactory,
-                  rpcControllerFactory, getMetaLookupPool(), 0)) {
+                  rpcControllerFactory, getMetaLookupPool(),
+                  metaReplicaCallTimeoutScanInMicroSecond)) {
             regionInfoRow = rcs.next();
           }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c03f003f/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 7d3190c..cbd5d55 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -309,6 +309,13 @@ public final class HConstants {
   /** Default HBase client operation timeout, which is tantamount to a blocking call */
   public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = 1200000;
 
+  /** Parameter name for HBase client meta replica scan call timeout. */
+  public static final String HBASE_CLIENT_MEAT_REPLICA_SCAN_TIMEOUT =
+      "hbase.client.meta.replica.scan.timeout";
+
+  /** Default HBase client meta replica scan call timeout, 1 second */
+  public static final int HBASE_CLIENT_MEAT_REPLICA_SCAN_TIMEOUT_DEFAULT = 1000000;
+
   /** Used to construct the name of the log directory for a region server */
   public static final String HREGION_LOGDIR_NAME = "WALs";
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c03f003f/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
index 892cff2..45a93a8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
@@ -74,6 +75,7 @@ public class TestReplicaWithCluster {
   private static final byte[] f = HConstants.CATALOG_FAMILY;
 
   private final static int REFRESH_PERIOD = 1000;
+  private final static int META_SCAN_TIMEOUT_IN_MILLISEC = 200;
 
   /**
    * This copro is used to synchronize the tests.
@@ -157,6 +159,34 @@ public class TestReplicaWithCluster {
     }
   }
 
+  /**
+   * This copro is used to slow down the primary meta region scan a bit
+   */
+  public static class RegionServerHostingPrimayMetaRegionSlowCopro extends BaseRegionObserver
{
+    static boolean slowDownPrimaryMetaScan = false;
+
+    @Override
+    public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment>
e,
+        final Scan scan, final RegionScanner s) throws IOException {
+
+      int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId();
+
+      // Slow down with the primary meta region scan
+      if (slowDownPrimaryMetaScan && (e.getEnvironment().getRegion().getRegionInfo().isMetaRegion()
+          && (replicaId == 0))) {
+        LOG.info("Scan with primary meta region, slow down a bit");
+        try {
+          Thread.sleep(META_SCAN_TIMEOUT_IN_MILLISEC - 50);
+        } catch (InterruptedException ie) {
+          // Ingore
+        }
+
+      }
+      return null;
+    }
+  }
+
+
   @BeforeClass
   public static void beforeClass() throws Exception {
     // enable store file refreshing
@@ -175,6 +205,19 @@ public class TestReplicaWithCluster {
     HTU.getConfiguration().setInt("hbase.client.primaryCallTimeout.get", 1000000);
     HTU.getConfiguration().setInt("hbase.client.primaryCallTimeout.scan", 1000000);
 
+    // Enable meta replica at server side
+    HTU.getConfiguration().setInt("hbase.meta.replica.count", 2);
+
+    // Make sure master does not host system tables.
+    HTU.getConfiguration().set("hbase.balancer.tablesOnMaster", "none");
+
+    // Set system coprocessor so it can be applied to meta regions
+    HTU.getConfiguration().set("hbase.coprocessor.region.classes",
+        RegionServerHostingPrimayMetaRegionSlowCopro.class.getName());
+
+    HTU.getConfiguration().setInt(HConstants.HBASE_CLIENT_MEAT_REPLICA_SCAN_TIMEOUT,
+        META_SCAN_TIMEOUT_IN_MILLISEC * 1000);
+
     HTU.startMiniCluster(NB_SERVERS);
     HTU.getHBaseCluster().startMaster();
   }
@@ -571,4 +614,37 @@ public class TestReplicaWithCluster {
       HTU.deleteTable(hdt.getTableName());
     }
   }
+
+  // This test is to test when hbase.client.metaReplicaCallTimeout.scan is configured, meta
table
+  // scan will always get the result from primary meta region as long as the result is returned
+  // within configured hbase.client.metaReplicaCallTimeout.scan from primary meta region.
+  @Test
+  public void testGetRegionLocationFromPrimaryMetaRegion() throws IOException, InterruptedException
{
+    HTU.getHBaseAdmin().setBalancerRunning(false, true);
+
+    ((ConnectionManager.HConnectionImplementation) HTU.getHBaseAdmin().getConnection()).
+        setUseMetaReplicas(true);
+
+    // Create table then get the single region for our new table.
+    HTableDescriptor hdt = HTU.createTableDescriptor("testGetRegionLocationFromPrimaryMetaRegion");
+    hdt.setRegionReplication(2);
+    try {
+
+      HTU.createTable(hdt, new byte[][] { f }, null);
+
+      RegionServerHostingPrimayMetaRegionSlowCopro.slowDownPrimaryMetaScan = true;
+
+      // Get user table location, always get it from the primary meta replica
+      RegionLocations url = ((ClusterConnection) HTU.getConnection())
+          .locateRegion(hdt.getTableName(), row, false, false);
+
+    } finally {
+      RegionServerHostingPrimayMetaRegionSlowCopro.slowDownPrimaryMetaScan = false;
+      ((ConnectionManager.HConnectionImplementation) HTU.getHBaseAdmin().getConnection()).
+          setUseMetaReplicas(false);
+      HTU.getHBaseAdmin().setBalancerRunning(true, true);
+      HTU.getHBaseAdmin().disableTable(hdt.getTableName());
+      HTU.deleteTable(hdt.getTableName());
+    }
+  }
 }


Mime
View raw message