hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject hadoop git commit: HDFS-7503. Namenode restart after large deletions can cause slow processReport. (Arpit Agarwal)
Date Thu, 11 Dec 2014 07:52:53 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-1 2dff54ebc -> a8eee1fd0


HDFS-7503. Namenode restart after large deletions can cause slow processReport. (Arpit Agarwal)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8eee1fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8eee1fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8eee1fd

Branch: refs/heads/branch-1
Commit: a8eee1fd01ddad153388e269b0d1381433864a82
Parents: 2dff54e
Author: arp <arp@apache.org>
Authored: Wed Dec 10 23:51:45 2014 -0800
Committer: arp <arp@apache.org>
Committed: Wed Dec 10 23:51:45 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                             |  3 +++
 .../hadoop/hdfs/server/namenode/FSNamesystem.java       | 12 +++++-------
 .../apache/hadoop/hdfs/server/namenode/NameNode.java    | 12 +++++++++++-
 3 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0bc2773..59bef6c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -256,6 +256,9 @@ Release 1.3.0 - unreleased
 
     MAPREDUCE-6170. TestUlimit failure on JDK8. (bc Wong via kasha)
 
+    HDFS-7503. Namenode restart after large deletions can cause slow
+    processReport (Arpit Agarwal)
+
 Release 1.2.2 - unreleased
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8a95a75..f9b9d7d 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3934,9 +3934,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean,
FSClusterSt
    * The given node is reporting all its blocks.  Use this info to 
    * update the (machine-->blocklist) and (block-->machinelist) tables.
    */
-  public synchronized void processReport(DatanodeID nodeID, 
-                                         BlockListAsLongs newReport
-                                        ) throws IOException {
+  public synchronized Collection<Block> processReport(
+      DatanodeID nodeID,
+      BlockListAsLongs newReport) throws IOException {
     long startTime = now();
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("BLOCK* processReport: "
@@ -3961,7 +3961,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean,
FSClusterSt
       NameNode.stateChangeLog.info("BLOCK* processReport: "
           + "discarded non-initial block report from " + nodeID.getName()
           + " because namenode still in startup phase");
-      return;
+      return null;
     }
 
     //
@@ -3980,9 +3980,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean,
FSClusterSt
       addStoredBlock(b, node, null);
     }
     for (Block b : toInvalidate) {
-      NameNode.stateChangeLog.info("BLOCK* processReport: " 
-          + b + " on " + node.getName() + " size " + b.getNumBytes()
-          + " does not belong to any file");
       addToInvalidates(b, node);
     }
     long endTime = now();
@@ -3991,6 +3988,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean,
FSClusterSt
         + nodeID.getName() + ", blocks: " + newReport.getNumberOfBlocks()
         + ", processing time: " + (endTime - startTime) + " msecs");
     node.processedBlockReport();
+    return toInvalidate;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8eee1fd/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 268d2e3..be0089c 100644
--- a/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -1130,7 +1130,17 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
     stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
            +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");
 
-    namesystem.processReport(nodeReg, blist);
+    Collection<Block> blocksInvalidated =
+        namesystem.processReport(nodeReg, blist);
+
+    final String node = nodeReg.toString();
+    if (blocksInvalidated != null) {
+      for (Block b : blocksInvalidated) {
+        stateChangeLog.info("BLOCK* processReport: "
+                                + b + " on " + node + " size " + b.getNumBytes()
+                                + " does not belong to any file");
+      }
+    }
     if (getFSImage().isUpgradeFinalized())
       return DatanodeCommand.FINALIZE;
     return null;


Mime
View raw message