hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1401136 - in /hadoop/common/branches/branch-1: ./ src/hdfs/ src/hdfs/org/apache/hadoop/hdfs/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/test/org/apache/hadoop/hdfs/server/namenode/
Date Tue, 23 Oct 2012 01:10:38 GMT
Author: szetszwo
Date: Tue Oct 23 01:10:37 2012
New Revision: 1401136

URL: http://svn.apache.org/viewvc?rev=1401136&view=rev
Log:
HDFS-3540. Change the default value of dfs.namenode.edits.toleration.length from -1 to 0,
i.e. enable the end of edit log check but do not tolerate any corruption.

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
    hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Oct 23 01:10:37 2012
@@ -97,6 +97,10 @@ Release 1.2.0 - unreleased
 
     HDFS-4057. NameNode.namesystem should be private. (Brandon Li via suresh)
 
+    HDFS-3540. Change the default value of dfs.namenode.edits.toleration.length
+    from -1 to 0, i.e. enable the end of edit log check but do not tolerate
+    any corruption.  (szetszwo)
+
   OPTIMIZATIONS
 
     HDFS-2533. Backport: Remove needless synchronization on some FSDataSet

Modified: hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml Tue Oct 23 01:10:37 2012
@@ -163,6 +163,25 @@ creations/deletions), or "all".</descrip
       directories, for redundancy. Default value is same as dfs.name.dir
   </description>
 </property>
+
+<property>
+  <name>dfs.namenode.edits.toleration.length</name>
+  <value>0</value>
+  <description>
+    The length in bytes that namenode is willing to tolerate when the edit log
+    is corrupted.  The edit log toleration feature checks the entire edit log.
+    It computes read length (the length of valid data), corruption length and
+    padding length.  In case that corruption length is non-zero, the corruption
+    will be tolerated only if the corruption length is less than or equal to
+    the toleration length.
+
+    For disabling edit log toleration feature, set this property to -1.  When
+    the feature is disabled, the end of edit log will not be checked.  In this
+    case, namenode will startup normally even if the end of edit log is
+    corrupted.
+  </description>
+</property>
+
 <property>
   <name>dfs.web.ugi</name>
   <value>webuser,webgroup</value>

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue
Oct 23 01:10:37 2012
@@ -81,7 +81,7 @@ public class DFSConfigKeys extends Commo
   public static final String  DFS_NAMENODE_CHECKPOINT_SIZE_KEY = "dfs.namenode.checkpoint.size";
   public static final long    DFS_NAMENODE_CHECKPOINT_SIZE_DEFAULT = 4194304;
   public static final String  DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY = "dfs.namenode.edits.toleration.length";
-  public static final int     DFS_NAMENODE_EDITS_TOLERATION_LENGTH_DEFAULT = -1; //i.e. disable
it
+  public static final int     DFS_NAMENODE_EDITS_TOLERATION_LENGTH_DEFAULT = 0;
   public static final String  DFS_NAMENODE_UPGRADE_PERMISSION_KEY = "dfs.namenode.upgrade.permission";
   public static final int     DFS_NAMENODE_UPGRADE_PERMISSION_DEFAULT = 00777;
   public static final String  DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY = "dfs.namenode.heartbeat.recheck-interval";

Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Tue Oct 23 01:10:37 2012
@@ -1357,6 +1357,21 @@ public class NameNode implements ClientP
         return;
       }
     }
+    final int tolerationLength = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY,
+        DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_DEFAULT);
+    if (tolerationLength >= 0) {
+      if (!confirmPrompt("You have selected Metadata Recovery mode and have set "
+          + DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY + " = "
+          + tolerationLength + ".  However, Metadata Recovery mode and the"
+          + " Edit Log Toleration feature cannot be enabled at the same time."
+          + "  Disable Edit Log Toleration? (Y/N)\n")) {
+        System.err.println("Recovery aborted at user request.\n");
+        return;
+      }
+      conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY, -1);
+    }
+
     MetaRecoveryContext.LOG.info("starting recovery...");
     Collection<File> namespaceDirs = FSNamesystem.getNamespaceDirs(conf);
     Collection<File> editDirs = 

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLogLoading.java
Tue Oct 23 01:10:37 2012
@@ -17,7 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.IOException;
@@ -26,6 +27,7 @@ import java.io.RandomAccessFile;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.Test;
 
@@ -38,6 +40,8 @@ public class TestEditLogLoading {
     // start a cluster
     Configuration conf = new Configuration();
     conf.set("dfs.name.dir", new File(MiniDFSCluster.getBaseDir(), "name").getPath());
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY, -1);
+
 
     MiniDFSCluster cluster = null;
     FileSystem fileSys = null;

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java?rev=1401136&r1=1401135&r2=1401136&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
Tue Oct 23 01:10:37 2012
@@ -18,22 +18,23 @@
 
 package org.apache.hadoop.hdfs.server.namenode;
 
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.util.List;
 
-import static org.junit.Assert.*;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.util.StringUtils;
 import org.junit.Test;
 
@@ -130,6 +131,7 @@ public class TestNameNodeRecovery {
   
     // Start up the mini dfs cluster
     Configuration conf = new Configuration();
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_EDITS_TOLERATION_LENGTH_KEY, -1);
     MiniDFSCluster cluster;
     cluster = new MiniDFSCluster(0, conf, 0, true, true, false,
         StartupOption.FORMAT, null, null, null);



Mime
View raw message