hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anto...@apache.org
Subject hbase git commit: HBASE-15354 Same criteria to clear metacache across operations (Ashu Pachauri)
Date Sat, 05 Mar 2016 02:02:04 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 664575598 -> 27cf0c8c3


HBASE-15354 Same criteria to clear metacache across operations (Ashu Pachauri)

Signed-off-by: Mikhail Antonov <antonov@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/27cf0c8c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/27cf0c8c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/27cf0c8c

Branch: refs/heads/master
Commit: 27cf0c8c365b61fe12a9e85422e30d1f217f6b7b
Parents: 6645755
Author: Ashu Pachauri <ashu210890@gmail.com>
Authored: Fri Feb 26 20:29:59 2016 -0800
Committer: Mikhail Antonov <antonov@apache.org>
Committed: Fri Mar 4 17:55:35 2016 -0800

----------------------------------------------------------------------
 .../client/RegionAdminServiceCallable.java      |  21 +-
 .../hbase/client/RegionServerCallable.java      |  21 +-
 .../hadoop/hbase/client/TestMetaCache.java      | 231 +++++++++++++++++++
 3 files changed, 237 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/27cf0c8c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java
index 189dbaa..8ff8b8b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionAdminServiceCallable.java
@@ -20,17 +20,13 @@ package org.apache.hadoop.hbase.client;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.net.ConnectException;
-import java.net.SocketTimeoutException;
 
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.exceptions.RegionMovedException;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -118,20 +114,9 @@ public abstract class RegionAdminServiceCallable<T> implements
RetryingCallable<
 
   @Override
   public void throwable(Throwable t, boolean retrying) {
-    if (t instanceof SocketTimeoutException ||
-        t instanceof ConnectException ||
-        t instanceof RetriesExhaustedException ||
-        (location != null && getConnection().isDeadServer(location.getServerName())))
{
-      // if thrown these exceptions, we clear all the cache entries that
-      // map to that slow/dead server; otherwise, let cache miss and ask
-      // hbase:meta again to find the new location
-      if (this.location != null) getConnection().clearCaches(location.getServerName());
-    } else if (t instanceof RegionMovedException) {
-      getConnection().updateCachedLocations(tableName, row, t, location);
-    } else if (t instanceof NotServingRegionException) {
-      // Purge cache entries for this specific region from hbase:meta cache
-      // since we don't call connect(true) when number of retries is 1.
-      getConnection().deleteCachedRegionLocation(location);
+    if (location != null) {
+      connection.updateCachedLocations(tableName, location.getRegionInfo().getRegionName(),
row,
+          t, location.getServerName());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/27cf0c8c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
index 9989d56..6bd8f75 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
@@ -20,17 +20,13 @@
 package org.apache.hadoop.hbase.client;
 
 import java.io.IOException;
-import java.net.ConnectException;
-import java.net.SocketTimeoutException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.exceptions.RegionMovedException;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -118,20 +114,9 @@ public abstract class RegionServerCallable<T> implements RetryingCallable<T>
{
 
   @Override
   public void throwable(Throwable t, boolean retrying) {
-    if (t instanceof SocketTimeoutException ||
-        t instanceof ConnectException ||
-        t instanceof RetriesExhaustedException ||
-        (location != null && getConnection().isDeadServer(location.getServerName())))
{
-      // if thrown these exceptions, we clear all the cache entries that
-      // map to that slow/dead server; otherwise, let cache miss and ask
-      // hbase:meta again to find the new location
-      if (this.location != null) getConnection().clearCaches(location.getServerName());
-    } else if (t instanceof RegionMovedException) {
-      getConnection().updateCachedLocations(tableName, row, t, location);
-    } else if (t instanceof NotServingRegionException && !retrying) {
-      // Purge cache entries for this specific region from hbase:meta cache
-      // since we don't call connect(true) when number of retries is 1.
-      getConnection().deleteCachedRegionLocation(location);
+    if (location != null) {
+      getConnection().updateCachedLocations(tableName, location.getRegionInfo().getRegionName(),
+          row, t, location.getServerName());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/27cf0c8c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
new file mode 100644
index 0000000..5738be9
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+
+import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
+import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
+import org.apache.hadoop.hbase.quotas.ThrottlingException;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RSRpcServices;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+@Category({MediumTests.class, ClientTests.class})
+public class TestMetaCache {
+  private static final Log LOG = LogFactory.getLog(TestMetaCache.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static final TableName TABLE_NAME = TableName.valueOf("test_table");
+  private static final byte[] FAMILY = Bytes.toBytes("fam1");
+  private static final byte[] QUALIFIER = Bytes.toBytes("qual");
+  private ConnectionImplementation conn;
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    conf.set("hbase.client.retries.number", "1");
+    conf.setStrings(HConstants.REGION_SERVER_IMPL, RegionServerWithFakeRpcServices.class.getName());
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @Before
+  public void setup() throws Exception {
+    conn = (ConnectionImplementation)ConnectionFactory.createConnection(
+        TEST_UTIL.getConfiguration());
+    HTableDescriptor table = new HTableDescriptor(TABLE_NAME);
+    HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
+    fam.setMaxVersions(2);
+    table.addFamily(fam);
+    try (Admin admin = conn.getAdmin()) {
+      admin.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+    }
+    TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @After
+  public void tearDown() throws Exception {
+    // Nothing to do.
+  }
+
+  @Test
+  public void testPreserveMetaCacheOnException() throws Exception {
+    Table table = conn.getTable(TABLE_NAME);
+    byte [] row = HBaseTestingUtility.KEYS[2];
+
+    Put put = new Put(row);
+    put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(10));
+    Get get = new Get(row);
+    Append append = new Append(row);
+    append.add(FAMILY, QUALIFIER, Bytes.toBytes(11));
+    Increment increment = new Increment(row);
+    increment.addColumn(FAMILY, QUALIFIER, 10);
+    Delete delete = new Delete(row);
+    delete.addColumn(FAMILY, QUALIFIER);
+    RowMutations mutations = new RowMutations(row);
+    mutations.add(put);
+    mutations.add(delete);
+
+    Exception exp;
+    boolean success;
+    for (int i = 0; i < 50; i++) {
+      exp = null;
+      success =false;
+      try {
+        table.put(put);
+        // If at least one operation succeeded, we should have cached the region location.
+        success = true;
+        table.get(get);
+        table.append(append);
+        table.increment(increment);
+        table.delete(delete);
+        table.mutateRow(mutations);
+      } catch (IOException ex) {
+        // Only keep track of the last exception that updated the meta cache
+        if (ClientExceptionsUtil.isMetaClearingException(ex) || success) {
+          exp = ex;
+        }
+      }
+      // Do not test if we did not touch the meta cache in this iteration.
+      if(exp != null && ClientExceptionsUtil.isMetaClearingException(exp)) {
+        assertNull(conn.getCachedLocation(TABLE_NAME, row));
+      } else if (success) {
+        assertNotNull(conn.getCachedLocation(TABLE_NAME, row));
+      }
+    }
+  }
+
+  public static List<Throwable> metaCachePreservingExceptions() {
+    return new ArrayList<Throwable>() {{
+        add(new RegionOpeningException(" "));
+        add(new RegionTooBusyException());
+        add(new ThrottlingException(" "));
+        add(new MultiActionResultTooLarge(" "));
+        add(new RetryImmediatelyException(" "));
+        add(new CallQueueTooBigException());
+    }};
+  }
+
+  protected static class RegionServerWithFakeRpcServices extends HRegionServer {
+
+    public RegionServerWithFakeRpcServices(Configuration conf, CoordinatedStateManager cp)
+        throws IOException, InterruptedException {
+      super(conf, cp);
+    }
+
+    @Override
+    protected RSRpcServices createRpcServices() throws IOException {
+      return new FakeRSRpcServices(this);
+    }
+  }
+
+  protected static class FakeRSRpcServices extends RSRpcServices {
+
+    private int numReqs = -1;
+    private int expCount = -1;
+    private List<Throwable> metaCachePreservingExceptions = metaCachePreservingExceptions();
+
+    public FakeRSRpcServices(HRegionServer rs) throws IOException {
+      super(rs);
+    }
+
+    @Override
+    public GetResponse get(final RpcController controller,
+        final ClientProtos.GetRequest request) throws ServiceException {
+      throwSomeExceptions();
+      return super.get(controller, request);
+    }
+
+    @Override
+    public ClientProtos.MutateResponse mutate(final RpcController controller,
+        final ClientProtos.MutateRequest request) throws ServiceException {
+      throwSomeExceptions();
+      return super.mutate(controller, request);
+    }
+
+    @Override
+    public ClientProtos.ScanResponse scan(final RpcController controller,
+        final ClientProtos.ScanRequest request) throws ServiceException {
+      throwSomeExceptions();
+      return super.scan(controller, request);
+    }
+
+    /**
+     * Throw some exceptions. Mostly throw exceptions which do not clear meta cache.
+     * Periodically throw NotSevingRegionException which clears the meta cache.
+     * @throws ServiceException
+     */
+    private void throwSomeExceptions() throws ServiceException {
+      numReqs++;
+      // Succeed every 5 request, throw cache clearing exceptions twice every 5 requests
and throw
+      // meta cache preserving exceptions otherwise.
+      if (numReqs % 5 ==0) {
+        return;
+      } else if (numReqs % 5 == 1 || numReqs % 5 == 2) {
+        throw new ServiceException(new NotServingRegionException());
+      }
+      // Round robin between different special exceptions.
+      // This is not ideal since exception types are not tied to the operation performed
here,
+      // But, we don't really care here if we throw MultiActionTooLargeException while doing
+      // single Gets.
+      expCount++;
+      Throwable t = metaCachePreservingExceptions.get(
+          expCount % metaCachePreservingExceptions.size());
+      throw new ServiceException(t);
+    }
+  }
+}


Mime
View raw message