hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject hadoop git commit: HDFS-9493. Test o.a.h.hdfs.server.namenode.TestMetaSave fails in trunk. (Tony Wu via lei)
Date Fri, 08 Jan 2016 23:11:01 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 40dabae23 -> 699d42049


HDFS-9493. Test o.a.h.hdfs.server.namenode.TestMetaSave fails in trunk.  (Tony Wu via lei)

(cherry picked from commit fd8065a763ff68db265ef23a7d4f97558e213ef9)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
	hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/699d4204
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/699d4204
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/699d4204

Branch: refs/heads/branch-2
Commit: 699d4204977cff31ac689457b4b99317e3bdc0d3
Parents: 40dabae
Author: Lei Xu <lei@apache.org>
Authored: Fri Jan 8 15:00:04 2016 -0800
Committer: Lei Xu <lei@apache.org>
Committed: Fri Jan 8 15:10:41 2016 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +
 .../blockmanagement/BlockManagerTestUtil.java   | 13 +++++
 .../hdfs/server/namenode/TestMetaSave.java      | 59 +++++++++++++++-----
 3 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/699d4204/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ebcc6aa..6797b51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1690,6 +1690,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-9626. TestBlockReplacement#testBlockReplacement fails occasionally.
     (Xiao Chen via zhz)
 
+    HDFS-9493. Test o.a.h.hdfs.server.namenode.TestMetaSave fails in trunk.
+    (Tony Wu via lei)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/699d4204/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 6e7b14e..3e41a33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -315,4 +315,17 @@ public class BlockManagerTestUtil {
     Whitebox.setInternalState(bmSafeMode, "extension", Integer.MAX_VALUE);
     Whitebox.setInternalState(bmSafeMode, "status", BMSafeModeStatus.EXTENSION);
   }
+
+  /**
+   * Check if a given Datanode (specified by uuid) is removed. Removed means the
+   * Datanode is no longer present in HeartbeatManager and NetworkTopology.
+   * @param nn Namenode
+   * @param dnUuid Datanode UUID
+   * @return true if datanode is removed.
+   */
+  public static boolean isDatanodeRemoved(NameNode nn, String dnUuid){
+      final DatanodeManager dnm =
+          nn.getNamesystem().getBlockManager().getDatanodeManager();
+      return !dnm.getNetworkTopology().contains(dnm.getDatanode(dnUuid));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/699d4204/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
index 786f83f..caca217 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
@@ -28,19 +28,25 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Random;
+import java.util.concurrent.TimeoutException;
 
+import com.google.common.base.Supplier;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 /**
@@ -65,8 +71,8 @@ public class TestMetaSave {
     stm.close();
   }
 
-  @BeforeClass
-  public static void setUp() throws IOException {
+  @Before
+  public void setUp() throws IOException {
     // start a cluster
     Configuration conf = new HdfsConfiguration();
 
@@ -75,6 +81,7 @@ public class TestMetaSave {
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
     conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
     conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1L);
+    conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 5L);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
     cluster.waitActive();
     fileSys = cluster.getFileSystem();
@@ -85,15 +92,16 @@ public class TestMetaSave {
    * Tests metasave
    */
   @Test
-  public void testMetaSave() throws IOException, InterruptedException {
+  public void testMetaSave()
+      throws IOException, InterruptedException, TimeoutException {
     for (int i = 0; i < 2; i++) {
       Path file = new Path("/filestatus" + i);
       createFile(fileSys, file);
     }
 
-    cluster.stopDataNode(1);
-    // wait for namenode to discover that a datanode is dead
-    Thread.sleep(15000);
+    // stop datanode and wait for namenode to discover that a datanode is dead
+    stopDatanodeAndWait(1);
+
     nnRpc.setReplication("/filestatus0", (short) 4);
 
     nnRpc.metaSave("metasave.out.txt");
@@ -126,15 +134,15 @@ public class TestMetaSave {
    */
   @Test
   public void testMetasaveAfterDelete()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, TimeoutException {
     for (int i = 0; i < 2; i++) {
       Path file = new Path("/filestatus" + i);
       createFile(fileSys, file);
     }
 
-    cluster.stopDataNode(1);
-    // wait for namenode to discover that a datanode is dead
-    Thread.sleep(15000);
+    // stop datanode and wait for namenode to discover that a datanode is dead
+    stopDatanodeAndWait(1);
+
     nnRpc.setReplication("/filestatus0", (short) 4);
     nnRpc.delete("/filestatus0", true);
     nnRpc.delete("/filestatus1", true);
@@ -211,8 +219,8 @@ public class TestMetaSave {
     }
   }
 
-  @AfterClass
-  public static void tearDown() throws IOException {
+  @After
+  public void tearDown() throws IOException {
     if (fileSys != null)
       fileSys.close();
     if (cluster != null)
@@ -228,4 +236,27 @@ public class TestMetaSave {
   private static File getLogFile(String name) {
     return new File(System.getProperty("hadoop.log.dir"), name);
   }
+
+  /**
+   * Stop a DN, notify NN the death of DN and wait for NN to remove the DN.
+   *
+   * @param dnIdx Index of the Datanode in MiniDFSCluster
+   * @throws TimeoutException
+   * @throws InterruptedException
+   */
+  private void stopDatanodeAndWait(final int dnIdx)
+      throws TimeoutException, InterruptedException {
+    final DataNode dnToStop = cluster.getDataNodes().get(dnIdx);
+    cluster.stopDataNode(dnIdx);
+    BlockManagerTestUtil.noticeDeadDatanode(
+        cluster.getNameNode(), dnToStop.getDatanodeId().getXferAddr());
+    // wait for namenode to discover that a datanode is dead
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        return BlockManagerTestUtil.isDatanodeRemoved(
+            cluster.getNameNode(), dnToStop.getDatanodeUuid());
+      }
+    }, 1000, 30000);
+  }
 }


Mime
View raw message