hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From whe...@apache.org
Subject hadoop git commit: HDFS-8273. FSNamesystem#Delete() should not call logSync() when holding the lock. Contributed by Haohui Mai.
Date Wed, 29 Apr 2015 01:06:25 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 30d0f1045 -> 4b9147ef8


HDFS-8273. FSNamesystem#Delete() should not call logSync() when holding the lock. Contributed
by Haohui Mai.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b9147ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b9147ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b9147ef

Branch: refs/heads/branch-2.7
Commit: 4b9147ef8c4f8885841b1e6c855f7ede5dc644c6
Parents: 30d0f10
Author: Haohui Mai <wheat9@apache.org>
Authored: Tue Apr 28 18:05:46 2015 -0700
Committer: Haohui Mai <wheat9@apache.org>
Committed: Tue Apr 28 18:06:22 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +++
 .../hdfs/server/namenode/FSDirDeleteOp.java     |  1 -
 .../hdfs/server/namenode/FSNamesystem.java      | 22 ++++++++++++--------
 3 files changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b9147ef/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e2be1aa..1a99e3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -50,6 +50,9 @@ Release 2.7.1 - UNRELEASED
     HDFS-8070. Pre-HDFS-7915 DFSClient cannot use short circuit on
     post-HDFS-7915 DataNode (cmccabe)
 
+    HDFS-8273. FSNamesystem#Delete() should not call logSync() when holding the
+    lock. (wheat9)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b9147ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
index 2fc4711..49a0c8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
@@ -158,7 +158,6 @@ class FSDirDeleteOp {
     incrDeletedFileCount(filesRemoved);
 
     fsn.removeLeasesAndINodes(src, removedINodes, true);
-    fsd.getEditLog().logSync();
 
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b9147ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4c55c76..94e7f6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3713,6 +3713,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     } finally {
       writeUnlock();
     }
+    getEditLog().logSync();
     if (toRemovedBlocks != null) {
       removeBlocks(toRemovedBlocks); // Incremental deletion of blocks
     }
@@ -4717,22 +4718,21 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
      * blocks and unlink them from the namespace.
      */
     private void clearCorruptLazyPersistFiles()
-        throws SafeModeException, AccessControlException,
-        UnresolvedLinkException, IOException {
+        throws IOException {
 
       BlockStoragePolicy lpPolicy = blockManager.getStoragePolicy("LAZY_PERSIST");
 
-      List<BlockCollection> filesToDelete = new ArrayList<BlockCollection>();
-
+      List<BlockCollection> filesToDelete = new ArrayList<>();
+      boolean changed = false;
       writeLock();
-
       try {
         final Iterator<Block> it = blockManager.getCorruptReplicaBlockIterator();
 
         while (it.hasNext()) {
           Block b = it.next();
           BlockInfoContiguous blockInfo = blockManager.getStoredBlock(b);
-          if (blockInfo.getBlockCollection().getStoragePolicyID() == lpPolicy.getId()) {
+          if (blockInfo.getBlockCollection().getStoragePolicyID()
+              == lpPolicy.getId()) {
             filesToDelete.add(blockInfo.getBlockCollection());
           }
         }
@@ -4740,9 +4740,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
         for (BlockCollection bc : filesToDelete) {
           LOG.warn("Removing lazyPersist file " + bc.getName() + " with no replicas.");
           BlocksMapUpdateInfo toRemoveBlocks =
-          FSDirDeleteOp.deleteInternal(
-              FSNamesystem.this, bc.getName(),
-              INodesInPath.fromINode((INodeFile) bc), false);
+              FSDirDeleteOp.deleteInternal(
+                  FSNamesystem.this, bc.getName(),
+                  INodesInPath.fromINode((INodeFile) bc), false);
+          changed |= toRemoveBlocks != null;
           if (toRemoveBlocks != null) {
             removeBlocks(toRemoveBlocks); // Incremental deletion of blocks
           }
@@ -4750,6 +4751,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       } finally {
         writeUnlock();
       }
+      if (changed) {
+        getEditLog().logSync();
+      }
     }
 
     @Override


Mime
View raw message