hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From weic...@apache.org
Subject hadoop git commit: HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.
Date Fri, 13 Apr 2018 19:19:00 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 994c7d66e -> ca8bb322b


HDFS-13330. ShortCircuitCache#fetchOrCreate never retries. Contributed by Gabor Bota.

(cherry picked from commit e66e287efe2b43e710137a628f03c7df3ebdf498)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca8bb322
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca8bb322
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca8bb322

Branch: refs/heads/branch-3.1
Commit: ca8bb322be5e21ce00bb49cecfc4f515b49d2a37
Parents: 994c7d6
Author: Wei-Chiu Chuang <weichiu@apache.org>
Authored: Fri Apr 13 09:17:34 2018 -0700
Committer: Wei-Chiu Chuang <weichiu@apache.org>
Committed: Fri Apr 13 12:18:49 2018 -0700

----------------------------------------------------------------------
 .../hdfs/shortcircuit/ShortCircuitCache.java    | 11 ++++++---
 .../shortcircuit/TestShortCircuitCache.java     | 26 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8bb322/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index b26652b..c2f0350 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -664,6 +664,7 @@ public class ShortCircuitCache implements Closeable {
     unref(replica);
   }
 
+  static final int FETCH_OR_CREATE_RETRY_TIMES = 3;
   /**
    * Fetch or create a replica.
    *
@@ -678,11 +679,11 @@ public class ShortCircuitCache implements Closeable {
    */
   public ShortCircuitReplicaInfo fetchOrCreate(ExtendedBlockId key,
       ShortCircuitReplicaCreator creator) {
-    Waitable<ShortCircuitReplicaInfo> newWaitable = null;
+    Waitable<ShortCircuitReplicaInfo> newWaitable;
     lock.lock();
     try {
       ShortCircuitReplicaInfo info = null;
-      do {
+      for (int i = 0; i < FETCH_OR_CREATE_RETRY_TIMES; i++){
         if (closed) {
           LOG.trace("{}: can't fethchOrCreate {} because the cache is closed.",
               this, key);
@@ -692,11 +693,12 @@ public class ShortCircuitCache implements Closeable {
         if (waitable != null) {
           try {
             info = fetch(key, waitable);
+            break;
           } catch (RetriableException e) {
             LOG.debug("{}: retrying {}", this, e.getMessage());
           }
         }
-      } while (false);
+      }
       if (info != null) return info;
       // We need to load the replica ourselves.
       newWaitable = new Waitable<>(lock.newCondition());
@@ -717,7 +719,8 @@ public class ShortCircuitCache implements Closeable {
    *
    * @throws RetriableException   If the caller needs to retry.
    */
-  private ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
+  @VisibleForTesting // ONLY for testing
+  protected ShortCircuitReplicaInfo fetch(ExtendedBlockId key,
       Waitable<ShortCircuitReplicaInfo> waitable) throws RetriableException {
     // Another thread is already in the process of loading this
     // ShortCircuitReplica.  So we simply wait for it to complete.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8bb322/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 7ba0edc..5da6a25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.ShortCircuitReplica
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -793,4 +794,29 @@ public class TestShortCircuitCache {
     cluster.shutdown();
     sockDir.close();
   }
+
+  @Test
+  public void testFetchOrCreateRetries() throws Exception {
+    try(ShortCircuitCache cache = Mockito
+        .spy(new ShortCircuitCache(10, 10000000, 10, 10000000, 1, 10000, 0))) {
+      final TestFileDescriptorPair pair = new TestFileDescriptorPair();
+      ExtendedBlockId extendedBlockId = new ExtendedBlockId(123, "test_bp1");
+      SimpleReplicaCreator sRC = new SimpleReplicaCreator(123, cache, pair);
+
+      // Arrange that fetch will throw RetriableException for any call
+      Mockito.doThrow(new RetriableException("Retry")).when(cache)
+        .fetch(Mockito.eq(extendedBlockId), Mockito.any());
+
+      // Act: calling fetchOrCreate two times
+      //  first call: it will create and put entry to replicaInfoMap
+      //  second call: it will call fetch to get info for entry, and should
+      //               retry 3 times because RetriableException thrown
+      cache.fetchOrCreate(extendedBlockId, sRC);
+      cache.fetchOrCreate(extendedBlockId, sRC);
+
+      // Assert that fetchOrCreate retried to fetch at least 3 times
+      Mockito.verify(cache, Mockito.atLeast(3))
+        .fetch(Mockito.eq(extendedBlockId), Mockito.any());
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message