Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B400CD789 for ; Mon, 4 Mar 2013 16:51:37 +0000 (UTC) Received: (qmail 57862 invoked by uid 500); 4 Mar 2013 16:51:37 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 57809 invoked by uid 500); 4 Mar 2013 16:51:37 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 57798 invoked by uid 99); 4 Mar 2013 16:51:37 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Mar 2013 16:51:37 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Mar 2013 16:51:32 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id C396223888CD; Mon, 4 Mar 2013 16:51:13 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1452391 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/resources/ src/test/java/org/apache/hadoop/hdfs/serve... Date: Mon, 04 Mar 2013 16:51:13 -0000 To: hdfs-commits@hadoop.apache.org From: daryn@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130304165113.C396223888CD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: daryn Date: Mon Mar 4 16:51:12 2013 New Revision: 1452391 URL: http://svn.apache.org/r1452391 Log: HDFS-4128. 2NN gets stuck in inconsistent state if edit log replay fails in the middle (kihwal via daryn) Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1452391&r1=1452390&r2=1452391&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Mar 4 16:51:12 2013 @@ -42,6 +42,9 @@ Release 0.23.7 - UNRELEASED HDFS-4495. Allow client-side lease renewal to be retried beyond soft-limit (kihwal) + HDFS-4128. 2NN gets stuck in inconsistent state if edit log replay fails + in the middle (kihwal via daryn) + Release 0.23.6 - 2013-02-06 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1452391&r1=1452390&r2=1452391&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon Mar 4 16:51:12 2013 @@ -96,6 +96,8 @@ public class DFSConfigKeys extends Commo public static final long DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT = 3600; public static final String DFS_NAMENODE_CHECKPOINT_TXNS_KEY = "dfs.namenode.checkpoint.txns"; public static final long DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT = 40000; + public static final String DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY = "dfs.namenode.checkpoint.max-retries"; + public static final int DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_DEFAULT = 3; public static final String DFS_NAMENODE_UPGRADE_PERMISSION_KEY = "dfs.namenode.upgrade.permission"; public static final int DFS_NAMENODE_UPGRADE_PERMISSION_DEFAULT = 00777; public static final String DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY = "dfs.namenode.heartbeat.recheck-interval"; Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1452391&r1=1452390&r2=1452391&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Mon Mar 4 16:51:12 2013 @@ -126,6 +126,9 @@ public class SecondaryNameNode implement /** checkpoint once every this many transactions, regardless of time */ private long checkpointTxnCount; + /** maxium number of retries when merge errors occur */ + private int maxRetries; + /** {@inheritDoc} */ public String toString() { @@ -144,6 +147,11 @@ public class SecondaryNameNode implement FSImage getFSImage() { return checkpointImage; } + + @VisibleForTesting + int getMergeErrorCount() { + return checkpointImage.getMergeErrorCount(); + } @VisibleForTesting void setFSImage(CheckpointStorage image) { @@ -239,6 +247,9 @@ public class SecondaryNameNode implement DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT); checkpointTxnCount = conf.getLong(DFS_NAMENODE_CHECKPOINT_TXNS_KEY, DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT); + + maxRetries = conf.getInt(DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY, + DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_DEFAULT); warnForDeprecatedConfigs(conf); // initialize the webserver for uploading files. @@ -398,6 +409,13 @@ public class SecondaryNameNode implement } catch (IOException e) { LOG.error("Exception in doCheckpoint", e); e.printStackTrace(); + // Prevent a huge number of edits from being created due to + // unrecoverable conditions and endless retries. + if (checkpointImage.getMergeErrorCount() > maxRetries) { + LOG.fatal("Merging failed " + + checkpointImage.getMergeErrorCount() + " times."); + terminate(1); + } } catch (Throwable e) { LOG.fatal("Throwable Exception in doCheckpoint", e); e.printStackTrace(); @@ -544,9 +562,20 @@ public class SecondaryNameNode implement RemoteEditLogManifest manifest = namenode.getEditLogManifest(sig.mostRecentCheckpointTxId + 1); + // Fetch fsimage and edits. Reload the image if previous merge failed. loadImage |= downloadCheckpointFiles( - fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits - doMerge(sig, manifest, loadImage, checkpointImage); + fsName, checkpointImage, sig, manifest) | + checkpointImage.hasMergeError(); + try { + doMerge(sig, manifest, loadImage, checkpointImage); + } catch (IOException ioe) { + // A merge error occurred. The in-memory file system state may be + // inconsistent, so the image and edits need to be reloaded. + checkpointImage.setMergeError(); + throw ioe; + } + // Clear any error since merge was successful. + checkpointImage.clearMergeError(); // // Upload the new image into the NameNode. Then tell the Namenode @@ -568,7 +597,7 @@ public class SecondaryNameNode implement // Since we've successfully checkpointed, we can remove some old // image files checkpointImage.purgeOldStorage(); - + return loadImage; } @@ -770,6 +799,7 @@ public class SecondaryNameNode implement } static class CheckpointStorage extends FSImage { + private int mergeErrorCount; /** * Construct a checkpoint image. * @param conf Node configuration. @@ -787,6 +817,7 @@ public class SecondaryNameNode implement // we shouldn't have any editLog instance. Setting to null // makes sure we don't accidentally depend on it. editLog = null; + mergeErrorCount = 0; } /** @@ -848,7 +879,24 @@ public class SecondaryNameNode implement } } } - + + + boolean hasMergeError() { + return (mergeErrorCount > 0); + } + + int getMergeErrorCount() { + return mergeErrorCount; + } + + void setMergeError() { + mergeErrorCount++; + } + + void clearMergeError() { + mergeErrorCount = 0; + } + /** * Ensure that the current/ directory exists in all storage * directories @@ -881,8 +929,14 @@ public class SecondaryNameNode implement dstImage.reloadFromImageFile(file); dstImage.getFSNamesystem().dir.imageLoadComplete(); } - Checkpointer.rollForwardByApplyingLogs(manifest, dstImage); + + // error simulation for junit tests + if (ErrorSimulator.getErrorSimulation(5)) { + throw new IOException("Simulating error5 " + + "after uploading new image to NameNode"); + } + dstImage.saveFSImageInAllDirs(dstImage.getLastAppliedTxId()); dstStorage.writeAll(); } Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1452391&r1=1452390&r2=1452391&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Mon Mar 4 16:51:12 2013 @@ -614,6 +614,16 @@ creations/deletions), or "all". + dfs.namenode.checkpoint.max-retries + 3 + The SecondaryNameNode retries failed checkpointing. If the + failure occurs while loading fsimage or replaying edits, the number of + retries is limited by this variable. + + + + + dfs.namenode.num.checkpoints.retained 2 The number of image checkpoint files that will be retained by Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1452391&r1=1452390&r2=1452391&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Mon Mar 4 16:51:12 2013 @@ -58,6 +58,8 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.ExitUtil.ExitException; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; import org.mockito.Mockito; @@ -94,7 +96,7 @@ public class TestCheckpoint extends Test @Override public void setUp() throws IOException { FileUtil.fullyDeleteContents(new File(MiniDFSCluster.getBaseDirectory())); - ErrorSimulator.initializeErrorSimulationEvent(5); + ErrorSimulator.initializeErrorSimulationEvent(6); } static void writeFile(FileSystem fileSys, Path name, int repl) @@ -201,6 +203,106 @@ public class TestCheckpoint extends Test } /* + * Simulate exception during edit replay. + */ + public void testReloadOnEditReplayFailure () throws IOException { + Configuration conf = new HdfsConfiguration(); + FSDataOutputStream fos = null; + SecondaryNameNode secondary = null; + MiniDFSCluster cluster = null; + FileSystem fs = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) + .build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + secondary = startSecondaryNameNode(conf); + fos = fs.create(new Path("tmpfile0")); + fos.write(new byte[] { 0, 1, 2, 3 }); + secondary.doCheckpoint(); + fos.write(new byte[] { 0, 1, 2, 3 }); + fos.hsync(); + + // Cause merge to fail in next checkpoint. + ErrorSimulator.setErrorSimulation(5); + + try { + secondary.doCheckpoint(); + fail("Fault injection failed."); + } catch (IOException ioe) { + // This is expected. + } + ErrorSimulator.clearErrorSimulation(5); + + // The error must be recorded, so next checkpoint will reload image. + fos.write(new byte[] { 0, 1, 2, 3 }); + fos.hsync(); + + assertTrue("Another checkpoint should have reloaded image", + secondary.doCheckpoint()); + } finally { + if (secondary != null) { + secondary.shutdown(); + } + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + ErrorSimulator.clearErrorSimulation(5); + } + } + + /* + * Simulate 2NN exit due to too many merge failures. + */ + public void testTooManyEditReplayFailures() throws IOException { + Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY, "1"); + conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, "1"); + + FSDataOutputStream fos = null; + SecondaryNameNode secondary = null; + MiniDFSCluster cluster = null; + FileSystem fs = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) + .build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fos = fs.create(new Path("tmpfile0")); + fos.write(new byte[] { 0, 1, 2, 3 }); + + // Cause merge to fail in next checkpoint. + secondary = startSecondaryNameNode(conf); + ErrorSimulator.setErrorSimulation(5); + secondary.doWork(); + // Fail if we get here. + fail("2NN did not exit."); + } catch (ExitException ee) { + // ignore + ExitUtil.resetFirstExitException(); + ExitUtil.clearTerminateCalled(); + assertEquals("Max retries", 1, secondary.getMergeErrorCount() - 1); + } finally { + if (secondary != null) { + secondary.shutdown(); + } + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + ErrorSimulator.clearErrorSimulation(5); + } + } + + + /* * Simulate namenode crashing after rolling edit log. */ public void testSecondaryNamenodeError1()