hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iva...@apache.org
Subject svn commit: r1491110 - in /hadoop/common/branches/branch-1-win: CHANGES.branch-1-win.txt src/hdfs/hdfs-default.xml src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Date Sun, 09 Jun 2013 00:20:53 GMT
Author: ivanmi
Date: Sun Jun  9 00:20:53 2013
New Revision: 1491110

URL: http://svn.apache.org/r1491110
Log:
HDFS-4677. Editlog should support synchronous writes. Contributed by Ivan Mitic.

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/src/hdfs/hdfs-default.xml
    hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
    hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1491110&r1=1491109&r2=1491110&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Sun Jun  9 00:20:53 2013
@@ -274,6 +274,8 @@ Branch-hadoop-1-win (branched from branc
     HADOOP-9552. Windows log4j template should suppress info messages from
     mortbay.log. (ivanmi)
 
+    HDFS-4677. Editlog should support synchronous writes. (ivanmi)
+
   Merged from branch-1
 
     HDFS-385. Backport: Add support for an experimental API that allows a

Modified: hadoop/common/branches/branch-1-win/src/hdfs/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/hdfs/hdfs-default.xml?rev=1491110&r1=1491109&r2=1491110&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/hdfs/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-1-win/src/hdfs/hdfs-default.xml Sun Jun  9 00:20:53 2013
@@ -463,4 +463,21 @@ creations/deletions), or "all".</descrip
   </description>
 </property>
 
+<property>
+  <name>dfs.namenode.edits.noeditlogchannelflush</name>
+  <value>false</value>
+  <description>
+    Specifies whether to flush edit log file channel. When set, expensive
+    FileChannel#force calls are skipped and synchronous disk writes are
+    enabled instead by opening the edit log file with RandomAccessFile("rws")
+    flags. This can significantly improve the performance of edit log writes
+    on the Windows platform.
+    Note that the behavior of the "rws" flags is platform and hardware specific
+    and might not provide the same level of guarantees as FileChannel#force.
+    For example, the write will skip the disk-cache on SAS and SCSI devices
+    while it might not on SATA devices. This is an expert level setting,
+    change with caution.
+  </description>
+</property>
+
 </configuration>

Modified: hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1491110&r1=1491109&r2=1491110&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++ hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
Sun Jun  9 00:20:53 2013
@@ -97,6 +97,8 @@ public class DFSConfigKeys extends Commo
   public static final String  DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT = "ssl-server.xml";
   public static final String  DFS_NAMENODE_NAME_DIR_RESTORE_KEY = "dfs.namenode.name.dir.restore";
   public static final boolean DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT = false;
+  public static final String  DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH = "dfs.namenode.edits.noeditlogchannelflush";
+  public static final boolean DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH_DEFAULT = false;
   public static final String  DFS_LIST_LIMIT = "dfs.ls.limit";
   public static final int     DFS_LIST_LIMIT_DEFAULT = 1000;
 

Modified: hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1491110&r1=1491109&r2=1491110&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/common/branches/branch-1-win/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Sun Jun  9 00:20:53 2013
@@ -38,6 +38,7 @@ import java.util.LinkedList;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -152,13 +153,25 @@ public class FSEditLog {
     private FileChannel fc;         // channel of the file stream for sync
     private DataOutputBuffer bufCurrent;  // current buffer for writing
     private DataOutputBuffer bufReady;    // buffer ready for flushing
+    private Configuration conf;
+    private boolean shouldSyncWritesAndSkipFsync = false;
 
     EditLogFileOutputStream(File name) throws IOException {
       super();
+      conf = new Configuration();
+      shouldSyncWritesAndSkipFsync = conf.getBoolean(
+          DFSConfigKeys.DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH,
+          DFSConfigKeys.DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH_DEFAULT);
+
       file = name;
       bufCurrent = new DataOutputBuffer(sizeFlushBuffer);
       bufReady = new DataOutputBuffer(sizeFlushBuffer);
-      RandomAccessFile rp = new RandomAccessFile(name, "rw");
+      RandomAccessFile rp;
+      if (shouldSyncWritesAndSkipFsync) {
+        rp = new RandomAccessFile(name, "rws");
+      } else {
+        rp = new RandomAccessFile(name, "rw");
+      }
       fp = new FileOutputStream(rp.getFD()); // open for append
       fc = rp.getChannel();
       fc.position(fc.size());
@@ -241,7 +254,10 @@ public class FSEditLog {
       preallocate();            // preallocate file if necessary
       bufReady.writeTo(fp);     // write data to file
       bufReady.reset();         // erase all data in the buffer
-      fc.force(false);          // metadata updates not needed because of preallocation
+      if (!shouldSyncWritesAndSkipFsync) {
+        // metadata updates not needed because of preallocation
+        fc.force(false);
+      }
     }
 
     /**



Mime
View raw message