hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject hadoop git commit: HDFS-7228. Fix TestDNFencing.testQueueingWithAppend. Contributed by Yongjun Zhang.
Date Fri, 07 Nov 2014 07:16:03 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 a2b46a734 -> 92337e1dc


HDFS-7228. Fix TestDNFencing.testQueueingWithAppend. Contributed by Yongjun Zhang.

(cherry picked from commit 75a9c894b3e53d34341e918c70ba7868d074e0fb)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92337e1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92337e1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92337e1d

Branch: refs/heads/branch-2.6
Commit: 92337e1dce57388d035002dd1478721170b490be
Parents: a2b46a7
Author: Jing Zhao <jing9@apache.org>
Authored: Wed Oct 22 10:47:49 2014 -0700
Committer: cnauroth <cnauroth@apache.org>
Committed: Thu Nov 6 23:12:08 2014 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  2 +
 .../hdfs/server/namenode/ha/TestDNFencing.java  | 49 +++++++++++++++++---
 2 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92337e1d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e5d667c..d53e3e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1105,6 +1105,8 @@ Release 2.6.0 - UNRELEASED
     dfs.namenode.secondary.http-address cannot be interpreted as a network
     address. (cnauroth)
 
+    HDFS-7226. Fix TestDNFencing.testQueueingWithAppend. (Yongjun Zhang via jing9)
+
 Release 2.5.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/92337e1d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index 66f301b..75d5b70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -415,6 +415,7 @@ public class TestDNFencing {
     int numQueued = 0;
     int numDN = cluster.getDataNodes().size();
     
+    // case 1: create file and call hflush after write
     FSDataOutputStream out = fs.create(TEST_FILE_PATH);
     try {
       AppendTestUtil.write(out, 0, 10);
@@ -422,29 +423,65 @@ public class TestDNFencing {
 
       // Opening the file will report RBW replicas, but will be
       // queued on the StandbyNode.
+      // However, the delivery of RBW messages is delayed by HDFS-7217 fix.
+      // Apply cluster.triggerBlockReports() to trigger the reporting sooner.
+      //
+      cluster.triggerBlockReports();
       numQueued += numDN; // RBW messages
+
+      // The cluster.triggerBlockReports() call above does a full 
+      // block report that incurs 3 extra RBW messages
+      numQueued += numDN; // RBW messages      
     } finally {
       IOUtils.closeStream(out);
       numQueued += numDN; // blockReceived messages
     }
-    
+
     cluster.triggerBlockReports();
     numQueued += numDN;
-    
+    assertEquals(numQueued, cluster.getNameNode(1).getNamesystem().
+        getPendingDataNodeMessageCount());
+
+    // case 2: append to file and call hflush after write
     try {
       out = fs.append(TEST_FILE_PATH);
       AppendTestUtil.write(out, 10, 10);
-      // RBW replicas once it's opened for append
-      numQueued += numDN;
+      out.hflush();
+      cluster.triggerBlockReports();
+      numQueued += numDN * 2; // RBW messages, see comments in case 1
+    } finally {
+      IOUtils.closeStream(out);
+      numQueued += numDN; // blockReceived
+    }
+    assertEquals(numQueued, cluster.getNameNode(1).getNamesystem().
+        getPendingDataNodeMessageCount());
 
+    // case 3: similar to case 2, except no hflush is called.
+    try {
+      out = fs.append(TEST_FILE_PATH);
+      AppendTestUtil.write(out, 20, 10);
     } finally {
+      // The write operation in the try block is buffered, thus no RBW message
+      // is reported yet until the closeStream call here. When closeStream is
+      // called, before HDFS-7217 fix, there would be three RBW messages
+      // (blockReceiving), plus three FINALIZED messages (blockReceived)
+      // delivered to NN. However, because of HDFS-7217 fix, the reporting of
+      // RBW  messages is postponed. In this case, they are even overwritten 
+      // by the blockReceived messages of the same block when they are waiting
+      // to be delivered. All this happens within the closeStream() call.
+      // What's delivered to NN is the three blockReceived messages. See 
+      //    BPServiceActor#addPendingReplicationBlockInfo 
+      //
       IOUtils.closeStream(out);
       numQueued += numDN; // blockReceived
     }
-    
+
     cluster.triggerBlockReports();
     numQueued += numDN;
 
+    LOG.info("Expect " + numQueued + " and got: " + cluster.getNameNode(1).getNamesystem().
+        getPendingDataNodeMessageCount());      
+
     assertEquals(numQueued, cluster.getNameNode(1).getNamesystem().
         getPendingDataNodeMessageCount());
 
@@ -458,7 +495,7 @@ public class TestDNFencing {
     assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
     assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
     
-    AppendTestUtil.check(fs, TEST_FILE_PATH, 20);
+    AppendTestUtil.check(fs, TEST_FILE_PATH, 30);
   }
   
   /**


Mime
View raw message