hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [1/3] hadoop git commit: HDFS-10642. TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently. (Contributed by Mingliang Liu)
Date Tue, 26 Jul 2016 19:49:35 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 0ee1644e3 -> c08c7b7a9
  refs/heads/branch-2.8 53bf63270 -> 45a6ee97b
  refs/heads/trunk da6adf515 -> d2cf8b54c


HDFS-10642. TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently.
(Contributed by Mingliang Liu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2cf8b54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2cf8b54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2cf8b54

Branch: refs/heads/trunk
Commit: d2cf8b54c546fad098307b771ffe40173700f5a8
Parents: da6adf5
Author: Arpit Agarwal <arp@apache.org>
Authored: Tue Jul 26 12:27:46 2016 -0700
Committer: Arpit Agarwal <arp@apache.org>
Committed: Tue Jul 26 12:27:46 2016 -0700

----------------------------------------------------------------------
 .../fsdataset/impl/LazyPersistTestCase.java     | 30 +++++++++++++++-----
 .../fsdataset/impl/TestLazyPersistFiles.java    |  9 +++---
 .../impl/TestLazyPersistLockedMemory.java       |  3 +-
 .../impl/TestLazyPersistReplicaPlacement.java   | 10 +++++--
 .../impl/TestLazyPersistReplicaRecovery.java    |  5 ++--
 5 files changed, 40 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index 3593e2d..799d5d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -130,17 +130,33 @@ public abstract class LazyPersistTestCase {
   public Timeout timeout = new Timeout(300000);
 
   protected final LocatedBlocks ensureFileReplicasOnStorageType(
-      Path path, StorageType storageType) throws IOException {
+      Path path, StorageType storageType)
+      throws IOException, TimeoutException, InterruptedException {
     // Ensure that returned block locations returned are correct!
     LOG.info("Ensure path: " + path + " is on StorageType: " + storageType);
     assertThat(fs.exists(path), is(true));
     long fileLength = client.getFileInfo(path.toString()).getLen();
-    LocatedBlocks locatedBlocks =
-        client.getLocatedBlocks(path.toString(), 0, fileLength);
-    for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
-      assertThat(locatedBlock.getStorageTypes()[0], is(storageType));
-    }
-    return locatedBlocks;
+
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        try {
+          LocatedBlocks locatedBlocks =
+              client.getLocatedBlocks(path.toString(), 0, fileLength);
+          for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
+            if (locatedBlock.getStorageTypes()[0] != storageType) {
+              return false;
+            }
+          }
+          return true;
+        } catch (IOException ioe) {
+          LOG.warn("Exception got in ensureFileReplicasOnStorageType()", ioe);
+          return false;
+        }
+      }
+    }, 100, 30 * 1000);
+
+    return client.getLocatedBlocks(path.toString(), 0, fileLength);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 950e9dc..8c43592 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -29,6 +29,7 @@ import java.util.EnumSet;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.hadoop.fs.StorageType.RAM_DISK;
@@ -89,7 +90,7 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
    */
   @Test
   public void testCorruptFilesAreDiscarded()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, TimeoutException {
     getClusterBuilder().setRamDiskReplicaCapacity(2).build();
     final String METHOD_NAME = GenericTestUtils.getMethodName();
     Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -123,7 +124,7 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
 
   @Test
   public void testDisableLazyPersistFileScrubber()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, TimeoutException {
     getClusterBuilder().setRamDiskReplicaCapacity(2).disableScrubber().build();
     final String METHOD_NAME = GenericTestUtils.getMethodName();
     Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -151,8 +152,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
   * If NN restarted then lazyPersist files should not deleted
   */
   @Test
-  public void testFileShouldNotDiscardedIfNNRestarted() throws IOException,
-      InterruptedException {
+  public void testFileShouldNotDiscardedIfNNRestarted()
+      throws IOException, InterruptedException, TimeoutException {
     getClusterBuilder().setRamDiskReplicaCapacity(2).build();
     final String METHOD_NAME = GenericTestUtils.getMethodName();
     Path path1 = new Path("/" + METHOD_NAME + ".01.dat");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
index eef8f0b..d154b1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
@@ -53,7 +53,8 @@ public class TestLazyPersistLockedMemory extends LazyPersistTestCase {
    * fall back to disk.
    */
   @Test
-  public void testWithNoLockedMemory() throws IOException {
+  public void testWithNoLockedMemory()
+      throws IOException, TimeoutException, InterruptedException {
     getClusterBuilder().setNumDatanodes(1)
                        .setMaxLockedMemory(0).build();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java
index c89475a..c16dbe5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.concurrent.TimeoutException;
 
 import static org.apache.hadoop.fs.StorageType.DEFAULT;
 import static org.apache.hadoop.fs.StorageType.RAM_DISK;
@@ -35,7 +36,8 @@ import static org.junit.Assert.fail;
 
 public class TestLazyPersistReplicaPlacement extends LazyPersistTestCase {
   @Test
-  public void testPlacementOnRamDisk() throws IOException {
+  public void testPlacementOnRamDisk()
+      throws IOException, TimeoutException, InterruptedException {
     getClusterBuilder().build();
     final String METHOD_NAME = GenericTestUtils.getMethodName();
     Path path = new Path("/" + METHOD_NAME + ".dat");
@@ -45,7 +47,8 @@ public class TestLazyPersistReplicaPlacement extends LazyPersistTestCase
{
   }
 
   @Test
-  public void testPlacementOnSizeLimitedRamDisk() throws IOException {
+  public void testPlacementOnSizeLimitedRamDisk()
+      throws IOException, TimeoutException, InterruptedException {
     getClusterBuilder().setRamDiskReplicaCapacity(3).build();
     final String METHOD_NAME = GenericTestUtils.getMethodName();
     Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
@@ -64,7 +67,8 @@ public class TestLazyPersistReplicaPlacement extends LazyPersistTestCase
{
    * @throws IOException
    */
   @Test
-  public void testFallbackToDisk() throws IOException {
+  public void testFallbackToDisk()
+      throws IOException, TimeoutException, InterruptedException {
     getClusterBuilder().setHasTransientStorage(false).build();
     final String METHOD_NAME = GenericTestUtils.getMethodName();
     Path path = new Path("/" + METHOD_NAME + ".dat");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2cf8b54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java
index 231353a..537f9e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.concurrent.TimeoutException;
 
 import static org.apache.hadoop.fs.StorageType.DEFAULT;
 import static org.apache.hadoop.fs.StorageType.RAM_DISK;
@@ -30,7 +31,7 @@ import static org.apache.hadoop.fs.StorageType.RAM_DISK;
 public class TestLazyPersistReplicaRecovery extends LazyPersistTestCase {
   @Test
   public void testDnRestartWithSavedReplicas()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, TimeoutException {
 
     getClusterBuilder().build();
     final String METHOD_NAME = GenericTestUtils.getMethodName();
@@ -55,7 +56,7 @@ public class TestLazyPersistReplicaRecovery extends LazyPersistTestCase
{
 
   @Test
   public void testDnRestartWithUnsavedReplicas()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, TimeoutException {
 
     getClusterBuilder().build();
     FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0));


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message