hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1360012 - in /hadoop/common/branches/branch-1.1: ./ src/hdfs/org/apache/hadoop/hdfs/server/datanode/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/mapred/
Date Wed, 11 Jul 2012 05:43:25 GMT
Author: eli
Date: Wed Jul 11 05:43:24 2012
New Revision: 1360012

URL: http://svn.apache.org/viewvc?rev=1360012&view=rev
Log:
HADOOP-8365. Add flag to disable durable sync. Contributed by Eli Collins

Modified:
    hadoop/common/branches/branch-1.1/   (props changed)
    hadoop/common/branches/branch-1.1/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
    hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
    hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
    hadoop/common/branches/branch-1.1/src/mapred/   (props changed)

Propchange: hadoop/common/branches/branch-1.1/
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1:r1360011

Modified: hadoop/common/branches/branch-1.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/CHANGES.txt?rev=1360012&r1=1360011&r2=1360012&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.1/CHANGES.txt Wed Jul 11 05:43:24 2012
@@ -123,6 +123,8 @@ Release 1.1.0 - 2012.07.09
 
     HDFS-3516. Check content-type in WebHdfsFileSystem.  (szetszwo)
 
+    HADOOP-8365. Provide ability to disable working sync. (eli)
+
   BUG FIXES
 
     MAPREDUCE-4087. [Gridmix] GenerateDistCacheData job of Gridmix can

Propchange: hadoop/common/branches/branch-1.1/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1/CHANGES.txt:r1360011

Modified: hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1360012&r1=1360011&r2=1360012&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++ hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Wed Jul 11 05:43:24 2012
@@ -228,6 +228,7 @@ public class DataNode extends Configured
   int socketWriteTimeout = 0;  
   boolean transferToAllowed = true;
   int writePacketSize = 0;
+  private boolean durableSync;
   boolean isBlockTokenEnabled;
   BlockTokenSecretManager blockTokenSecretManager;
   boolean isBlockTokenInitialized = false;
@@ -294,6 +295,7 @@ public class DataNode extends Configured
         DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
 
     datanodeObject = this;
+    durableSync = conf.getBoolean("dfs.durable.sync", true);
     this.userWithLocalPathAccess = conf
         .get(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY);
     try {
@@ -755,11 +757,12 @@ public class DataNode extends Configured
       dnRegistration.exportedKeys = ExportedBlockKeys.DUMMY_KEYS;
     }
 
-
-    Block[] bbwReport = data.getBlocksBeingWrittenReport();
-    long[] blocksBeingWritten =
-      BlockListAsLongs.convertToArrayLongs(bbwReport);
-    namenode.blocksBeingWrittenReport(dnRegistration, blocksBeingWritten);
+    if (durableSync) {
+      Block[] bbwReport = data.getBlocksBeingWrittenReport();
+      long[] blocksBeingWritten =
+        BlockListAsLongs.convertToArrayLongs(bbwReport);
+      namenode.blocksBeingWrittenReport(dnRegistration, blocksBeingWritten);
+    }
 
     // random short delay - helps scatter the BR from all DNs
     // - but we can start generating the block report immediately

Modified: hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=1360012&r1=1360011&r2=1360012&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
(original)
+++ hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
Wed Jul 11 05:43:24 2012
@@ -347,6 +347,7 @@ public class FSDataset implements FSCons
       this.reserved = conf.getLong("dfs.datanode.du.reserved", 0);
       this.dataDir = new FSDir(currentDir);
       this.currentDir = currentDir;
+      boolean durableSync = conf.getBoolean("dfs.durable.sync", true);
       File parent = currentDir.getParentFile();
 
       this.detachDir = new File(parent, "detach");
@@ -366,7 +367,11 @@ public class FSDataset implements FSCons
       // should not be deleted.
       blocksBeingWritten = new File(parent, "blocksBeingWritten");
       if (blocksBeingWritten.exists()) {
-        recoverBlocksBeingWritten(blocksBeingWritten);
+        if (durableSync) {  
+          recoverBlocksBeingWritten(blocksBeingWritten);
+        } else {
+          FileUtil.fullyDelete(blocksBeingWritten);
+        }
       }
       
       if (!blocksBeingWritten.mkdirs()) {

Modified: hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1360012&r1=1360011&r2=1360012&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++ hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Wed Jul 11 05:43:24 2012
@@ -316,6 +316,8 @@ public class FSNamesystem implements FSC
   private long defaultBlockSize = 0;
   // allow file appending (for test coverage)
   private boolean allowBrokenAppend = false;
+  // enable durable sync
+  private boolean durableSync = true;
 
   /**
    * Last block index used for replication work.
@@ -538,6 +540,11 @@ public class FSNamesystem implements FSC
                "however append is not supported. This configuration option " +
                "is no longer required to enable sync.");
     }
+    this.durableSync = conf.getBoolean("dfs.durable.sync", true);
+    if (!durableSync) {
+      LOG.warn("Durable sync disabled. Beware data loss when running " +
+               "programs like HBase that require durable sync!");
+    }
     this.isAccessTokenEnabled = conf.getBoolean(
         DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, false);
     if (isAccessTokenEnabled) {
@@ -2312,11 +2319,13 @@ public class FSNamesystem implements FSC
     }
 
     // If this commit does not want to close the file, persist
-    // blocks and return
+    // blocks (if durable sync is enabled) and return
     src = leaseManager.findPath(pendingFile);
     if (!closeFile) {
-      dir.persistBlocks(src, pendingFile);
-      getEditLog().logSync();
+      if (durableSync) {
+        dir.persistBlocks(src, pendingFile);
+        getEditLog().logSync();
+      }
       LOG.info("commitBlockSynchronization(" + lastblock + ") successful");
       return;
     }

Propchange: hadoop/common/branches/branch-1.1/src/mapred/
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-1/src/mapred:r1360011



Mime
View raw message