hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: HDFS-12603. Enable async edit logging by default.
Date Mon, 09 Oct 2017 18:28:56 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 4d377c89b -> 42d3229e6


HDFS-12603. Enable async edit logging by default.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42d3229e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42d3229e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42d3229e

Branch: refs/heads/branch-2
Commit: 42d3229e6f604d13a0419ac32979500dae306066
Parents: 4d377c8
Author: Andrew Wang <wang@apache.org>
Authored: Mon Oct 9 11:21:43 2017 -0700
Committer: Andrew Wang <wang@apache.org>
Committed: Mon Oct 9 11:21:43 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  7 ++++++-
 .../hdfs/server/namenode/FSEditLogAsync.java    |  5 +++++
 .../src/main/resources/hdfs-default.xml         |  2 +-
 .../namenode/ha/TestFailureToReadEdits.java     | 22 +++++++++++++-------
 5 files changed, 28 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d3229e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e3ce0ab..6bec228 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -286,7 +286,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
 
   public static final String  DFS_NAMENODE_EDITS_ASYNC_LOGGING =
       "dfs.namenode.edits.asynclogging";
-  public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
+  public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
 
   public static final String  DFS_LIST_LIMIT = "dfs.ls.limit";
   public static final int     DFS_LIST_LIMIT_DEFAULT = 1000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d3229e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 2ab15fd..7c1b9b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -126,7 +126,7 @@ public class FSEditLog implements LogsPurgeable {
    * 
    * In a non-HA setup:
    * 
-   * The log starts in UNITIALIZED state upon construction. Once it's
+   * The log starts in UNINITIALIZED state upon construction. Once it's
    * initialized, it is usually in IN_SEGMENT state, indicating that edits may
    * be written. In the middle of a roll, or while saving the namespace, it
    * briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous
@@ -1746,4 +1746,9 @@ public class FSEditLog implements LogsPurgeable {
     }
     return count;
   }
+
+  @Override
+  public String toString() {
+    return super.toString();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d3229e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
index c14a310..1f5dc75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
@@ -319,4 +319,9 @@ class FSEditLogAsync extends FSEditLog implements Runnable {
       return "["+getClass().getSimpleName()+" op:"+op+" call:"+call+"]";
     }
   }
+
+  @Override
+  public String toString() {
+    return super.toString();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d3229e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index ccb8aff..94183e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3848,7 +3848,7 @@
 
 <property>
   <name>dfs.namenode.edits.asynclogging</name>
-  <value>false</value>
+  <value>true</value>
   <description>
     If set to true, enables asynchronous edit logs in the Namenode.  If set
     to false, the Namenode uses the traditional synchronous edit logs.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d3229e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index 5221ef9..26a0c19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
   private static final Random RANDOM = new Random();
 
   private final TestType clusterType;
+  private final boolean useAsyncEditLogging;
   private Configuration conf;
   private MiniDFSCluster cluster;
   private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@@ -88,18 +89,23 @@ public class TestFailureToReadEdits {
   };
   
   /**
-   * Run this suite of tests both for QJM-based HA and for file-based
-   * HA.
+   * Run this suite of tests for {QJM-based, file-based HA} x {async
+   * edit logging enabled, disabled}.
    */
   @Parameters
   public static Iterable<Object[]> data() {
-    return Arrays.asList(new Object[][] {
-        { TestType.SHARED_DIR_HA },
-        { TestType.QJM_HA } });
+    return Arrays.asList(new Object[][]{
+        {TestType.SHARED_DIR_HA, Boolean.FALSE},
+        {TestType.SHARED_DIR_HA, Boolean.TRUE},
+        {TestType.QJM_HA, Boolean.FALSE},
+        {TestType.QJM_HA, Boolean.TRUE},
+    });
   }
-  
-  public TestFailureToReadEdits(TestType clusterType) {
+
+  public TestFailureToReadEdits(TestType clusterType, Boolean
+      useAsyncEditLogging) {
     this.clusterType = clusterType;
+    this.useAsyncEditLogging = useAsyncEditLogging;
   }
 
   @Before
@@ -109,6 +115,8 @@ public class TestFailureToReadEdits {
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
     conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+        useAsyncEditLogging);
     HAUtil.setAllowStandbyReads(conf, true);
 
     if (clusterType == TestType.SHARED_DIR_HA) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message