hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [02/50] hadoop git commit: HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang.
Date Tue, 24 Oct 2017 00:40:54 GMT
HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/035c6ee5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/035c6ee5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/035c6ee5

Branch: refs/heads/HDFS-7240
Commit: 035c6ee587e444550af6420676e4cee049e09869
Parents: 9fcc3a1
Author: Xiao Chen <xiao@apache.org>
Authored: Mon Oct 16 09:43:39 2017 -0700
Committer: Xiao Chen <xiao@apache.org>
Committed: Mon Oct 16 09:51:10 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  2 +-
 .../src/main/resources/hdfs-default.xml         |  2 +-
 .../namenode/ha/TestFailureToReadEdits.java     | 25 ++++++++++++++------
 4 files changed, 21 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 1f96763..395b192 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
 
   public static final String  DFS_NAMENODE_EDITS_ASYNC_LOGGING =
       "dfs.namenode.edits.asynclogging";
-  public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
+  public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
 
   public static final String  DFS_LIST_LIMIT = "dfs.ls.limit";
   public static final int     DFS_LIST_LIMIT_DEFAULT = 1000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index af2a5af9..7ca63f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
    * 
    * In a non-HA setup:
    * 
-   * The log starts in UNITIALIZED state upon construction. Once it's
+   * The log starts in UNINITIALIZED state upon construction. Once it's
    * initialized, it is usually in IN_SEGMENT state, indicating that edits may
    * be written. In the middle of a roll, or while saving the namespace, it
    * briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous

http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3491ed2..3752578 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4105,7 +4105,7 @@
 
 <property>
   <name>dfs.namenode.edits.asynclogging</name>
-  <value>false</value>
+  <value>true</value>
   <description>
     If set to true, enables asynchronous edit logs in the Namenode.  If set
     to false, the Namenode uses the traditional synchronous edit logs.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index 93c717c..a37631f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
   private static final Random RANDOM = new Random();
 
   private final TestType clusterType;
+  private final boolean useAsyncEditLogging;
   private Configuration conf;
   private MiniDFSCluster cluster;
   private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@@ -88,18 +89,26 @@ public class TestFailureToReadEdits {
   };
   
   /**
-   * Run this suite of tests both for QJM-based HA and for file-based
-   * HA.
+   * Run this suite of tests for {QJM-based, file-based HA} x {async
+   * edit logging enabled, disabled}.
+   *
+   * TODO: Enable the test cases with async edit logging on. See HDFS-12603
+   * and HDFS-12660.
    */
   @Parameters
   public static Iterable<Object[]> data() {
-    return Arrays.asList(new Object[][] {
-        { TestType.SHARED_DIR_HA },
-        { TestType.QJM_HA } });
+    return Arrays.asList(new Object[][]{
+        {TestType.SHARED_DIR_HA, Boolean.FALSE},
+        //{TestType.SHARED_DIR_HA, Boolean.TRUE},
+        {TestType.QJM_HA, Boolean.FALSE},
+        //{TestType.QJM_HA, Boolean.TRUE},
+    });
   }
-  
-  public TestFailureToReadEdits(TestType clusterType) {
+
+  public TestFailureToReadEdits(TestType clusterType, Boolean
+      useAsyncEditLogging) {
     this.clusterType = clusterType;
+    this.useAsyncEditLogging = useAsyncEditLogging;
   }
 
   @Before
@@ -109,6 +118,8 @@ public class TestFailureToReadEdits {
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
     conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+        useAsyncEditLogging);
     HAUtil.setAllowStandbyReads(conf, true);
 
     if (clusterType == TestType.SHARED_DIR_HA) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message