hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject git commit: HDFS-6773. MiniDFSCluster should skip edit log fsync by default. Contributed by Stephen Chu.
Date Wed, 27 Aug 2014 20:38:01 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 99d843451 -> ddb9d565c


HDFS-6773. MiniDFSCluster should skip edit log fsync by default.  Contributed by Stephen Chu.

(cherry picked from commit d805cc27a98abbdf14a20ef3127a2c7cb212c765)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddb9d565
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddb9d565
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddb9d565

Branch: refs/heads/branch-2
Commit: ddb9d565c8b8186ab72c806291076c21f328bb6f
Parents: 99d8434
Author: Colin Patrick Mccabe <cmccabe@cloudera.com>
Authored: Wed Aug 27 13:33:02 2014 -0700
Committer: Colin Patrick Mccabe <cmccabe@cloudera.com>
Committed: Wed Aug 27 13:37:48 2014 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +++
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  | 22 +++++++++++++++++---
 .../server/datanode/TestFsDatasetCache.java     |  1 -
 .../server/namenode/TestCacheDirectives.java    |  1 -
 4 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddb9d565/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 29654cf..0f12bd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -166,6 +166,9 @@ Release 2.6.0 - UNRELEASED
 
     HDFS-6690. Deduplicate xattr names in memory. (wang)
 
+    HDFS-6773. MiniDFSCluster should skip edit log fsync by default (Stephen
+    Chu via Colin Patrick McCabe)
+
   BUG FIXES
 
     HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddb9d565/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 2cc7835..e7577be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -94,6 +94,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
+import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@@ -174,6 +175,7 @@ public class MiniDFSCluster {
     private boolean checkDataNodeAddrConfig = false;
     private boolean checkDataNodeHostConfig = false;
     private Configuration[] dnConfOverlays;
+    private boolean skipFsyncForTesting = true;
     
     public Builder(Configuration conf) {
       this.conf = conf;
@@ -407,6 +409,15 @@ public class MiniDFSCluster {
       this.dnConfOverlays = dnConfOverlays;
       return this;
     }
+
+    /**
+     * Default: true
+     * When true, we skip fsync() calls for speed improvements.
+     */
+    public Builder skipFsyncForTesting(boolean val) {
+      this.skipFsyncForTesting = val;
+      return this;
+    }
     
     /**
      * Construct the actual MiniDFSCluster
@@ -474,7 +485,8 @@ public class MiniDFSCluster {
                        builder.checkExitOnShutdown,
                        builder.checkDataNodeAddrConfig,
                        builder.checkDataNodeHostConfig,
-                       builder.dnConfOverlays);
+                       builder.dnConfOverlays,
+                       builder.skipFsyncForTesting);
   }
   
   public class DataNodeProperties {
@@ -727,7 +739,8 @@ public class MiniDFSCluster {
                        manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
                        operation, null, racks, hosts,
                        null, simulatedCapacities, null, true, false,
-                       MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false,
null);
+                       MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0),
+                       true, false, false, null, true);
   }
 
   private void initMiniDFSCluster(
@@ -742,7 +755,8 @@ public class MiniDFSCluster {
       MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
       boolean checkDataNodeAddrConfig,
       boolean checkDataNodeHostConfig,
-      Configuration[] dnConfOverlays)
+      Configuration[] dnConfOverlays,
+      boolean skipFsyncForTesting)
   throws IOException {
     boolean success = false;
     try {
@@ -785,6 +799,8 @@ public class MiniDFSCluster {
             + "Standby node since no IPC ports have been specified.");
         conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1);
       }
+
+      EditLogFileOutputStream.setShouldSkipFsyncForTesting(skipFsyncForTesting);
     
       federation = nnTopology.isFederated();
       try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddb9d565/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
index 5ac13ee..d6e70d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
@@ -108,7 +108,6 @@ public class TestFsDatasetCache {
   private static CacheManipulator prevCacheManipulator;
 
   static {
-    EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
     LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddb9d565/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
index d54b90e..9307692 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
@@ -110,7 +110,6 @@ public class TestCacheDirectives {
 
   static {
     NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());
-    EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
   }
 
   private static final long BLOCK_SIZE = 4096;


Mime
View raw message