Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 470C3200D35 for ; Tue, 24 Oct 2017 02:40:56 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 459ED160BF2; Tue, 24 Oct 2017 00:40:56 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 65166160BF0 for ; Tue, 24 Oct 2017 02:40:55 +0200 (CEST) Received: (qmail 60101 invoked by uid 500); 24 Oct 2017 00:40:54 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 60083 invoked by uid 99); 24 Oct 2017 00:40:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Oct 2017 00:40:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7EBDFDFC40; Tue, 24 Oct 2017 00:40:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Tue, 24 Oct 2017 00:40:54 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [02/50] hadoop git commit: HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang. archived-at: Tue, 24 Oct 2017 00:40:56 -0000 HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/035c6ee5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/035c6ee5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/035c6ee5 Branch: refs/heads/HDFS-7240 Commit: 035c6ee587e444550af6420676e4cee049e09869 Parents: 9fcc3a1 Author: Xiao Chen Authored: Mon Oct 16 09:43:39 2017 -0700 Committer: Xiao Chen Committed: Mon Oct 16 09:51:10 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 2 +- .../src/main/resources/hdfs-default.xml | 2 +- .../namenode/ha/TestFailureToReadEdits.java | 25 ++++++++++++++------ 4 files changed, 21 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 1f96763..395b192 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING = "dfs.namenode.edits.asynclogging"; - public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false; + public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true; public static final String DFS_LIST_LIMIT = "dfs.ls.limit"; public static final int DFS_LIST_LIMIT_DEFAULT = 1000; http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index af2a5af9..7ca63f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable { * * In a non-HA setup: * - * The log starts in UNITIALIZED state upon construction. Once it's + * The log starts in UNINITIALIZED state upon construction. Once it's * initialized, it is usually in IN_SEGMENT state, indicating that edits may * be written. In the middle of a roll, or while saving the namespace, it * briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 3491ed2..3752578 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4105,7 +4105,7 @@ dfs.namenode.edits.asynclogging - false + true If set to true, enables asynchronous edit logs in the Namenode. If set to false, the Namenode uses the traditional synchronous edit logs. http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java index 93c717c..a37631f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java @@ -75,6 +75,7 @@ public class TestFailureToReadEdits { private static final Random RANDOM = new Random(); private final TestType clusterType; + private final boolean useAsyncEditLogging; private Configuration conf; private MiniDFSCluster cluster; private MiniQJMHACluster miniQjmHaCluster; // for QJM case only @@ -88,18 +89,26 @@ public class TestFailureToReadEdits { }; /** - * Run this suite of tests both for QJM-based HA and for file-based - * HA. + * Run this suite of tests for {QJM-based, file-based HA} x {async + * edit logging enabled, disabled}. + * + * TODO: Enable the test cases with async edit logging on. See HDFS-12603 + * and HDFS-12660. */ @Parameters public static Iterable data() { - return Arrays.asList(new Object[][] { - { TestType.SHARED_DIR_HA }, - { TestType.QJM_HA } }); + return Arrays.asList(new Object[][]{ + {TestType.SHARED_DIR_HA, Boolean.FALSE}, + //{TestType.SHARED_DIR_HA, Boolean.TRUE}, + {TestType.QJM_HA, Boolean.FALSE}, + //{TestType.QJM_HA, Boolean.TRUE}, + }); } - - public TestFailureToReadEdits(TestType clusterType) { + + public TestFailureToReadEdits(TestType clusterType, Boolean + useAsyncEditLogging) { this.clusterType = clusterType; + this.useAsyncEditLogging = useAsyncEditLogging; } @Before @@ -109,6 +118,8 @@ public class TestFailureToReadEdits { conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING, + useAsyncEditLogging); HAUtil.setAllowStandbyReads(conf, true); if (clusterType == TestType.SHARED_DIR_HA) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org