From hdfs-commits-return-1882-apmail-hadoop-hdfs-commits-archive=hadoop.apache.org@hadoop.apache.org Tue Jul 19 23:04:20 2011 Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 17ED66525 for ; Tue, 19 Jul 2011 23:04:20 +0000 (UTC) Received: (qmail 84934 invoked by uid 500); 19 Jul 2011 23:04:19 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 84853 invoked by uid 500); 19 Jul 2011 23:04:19 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 84844 invoked by uid 99); 19 Jul 2011 23:04:18 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jul 2011 23:04:18 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jul 2011 23:04:17 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 722C523888EA; Tue, 19 Jul 2011 23:03:57 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1148580 - in /hadoop/common/branches/HDFS-1073/hdfs: ./ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/ Date: Tue, 19 Jul 2011 23:03:57 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110719230357.722C523888EA@eris.apache.org> Author: todd Date: Tue Jul 19 23:03:54 2011 New Revision: 1148580 URL: http://svn.apache.org/viewvc?rev=1148580&view=rev Log: HDFS-2168. Reenable TestEditLog.testFailedOpen and fix exposed bug. Contributed by Todd Lipcon. Modified: hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Modified: hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt?rev=1148580&r1=1148579&r2=1148580&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt (original) +++ hadoop/common/branches/HDFS-1073/hdfs/CHANGES.HDFS-1073.txt Tue Jul 19 23:03:54 2011 @@ -76,3 +76,4 @@ HDFS-2104. Add a flag to the 2NN to form (todd) HDFS-2135. Fix regression of HDFS-1955 in HDFS-1073 branch. (todd) HDFS-2160. Fix CreateEditsLog test tool in HDFS-1073 branch. (todd) +HDFS-2168. Reenable TestEditLog.testFailedOpen and fix exposed bug. (todd) Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java?rev=1148580&r1=1148579&r2=1148580&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java Tue Jul 19 23:03:54 2011 @@ -310,7 +310,8 @@ public class BackupImage extends FSImage * This causes the BN to also start the new edit log in its local * directories. */ - synchronized void namenodeStartedLogSegment(long txid) { + synchronized void namenodeStartedLogSegment(long txid) + throws IOException { LOG.info("NameNode started a new log segment at txid " + txid); if (editLog.isOpen()) { if (editLog.getLastWrittenTxId() == txid - 1) { Modified: hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1148580&r1=1148579&r2=1148580&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Tue Jul 19 23:03:54 2011 @@ -726,6 +726,19 @@ public class FSEditLog { } /** + * @return the number of active (non-failed) journals + */ + private int countActiveJournals() { + int count = 0; + for (JournalAndStream jas : journals) { + if (jas.isActive()) { + count++; + } + } + return count; + } + + /** * Used only by unit tests. */ @VisibleForTesting @@ -777,7 +790,7 @@ public class FSEditLog { * Transitions from BETWEEN_LOG_SEGMENTS state to IN_LOG_SEGMENT state. */ synchronized void startLogSegment(final long segmentTxId, - boolean writeHeaderTxn) { + boolean writeHeaderTxn) throws IOException { LOG.info("Starting log segment at " + segmentTxId); Preconditions.checkArgument(segmentTxId > 0, "Bad txid: %s", segmentTxId); @@ -789,7 +802,6 @@ public class FSEditLog { Preconditions.checkArgument(segmentTxId == txid + 1, "Cannot start log segment at txid %s when next expected " + "txid is %s", segmentTxId, txid + 1); - curSegmentTxId = segmentTxId; numTransactions = totalTimeTransactions = numTransactionsBatchedInSync = 0; @@ -803,6 +815,12 @@ public class FSEditLog { } }, "starting log segment " + segmentTxId); + if (countActiveJournals() == 0) { + throw new IOException("Unable to start log segment " + + segmentTxId + ": no journals successfully started."); + } + + curSegmentTxId = segmentTxId; state = State.IN_SEGMENT; if (writeHeaderTxn) { Modified: hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1148580&r1=1148579&r2=1148580&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/common/branches/HDFS-1073/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Tue Jul 19 23:03:54 2011 @@ -726,55 +726,20 @@ public class TestEditLog extends TestCas } } - /* - * TODO: need to re-enable this test in HDFS-1073 branch! public void testFailedOpen() throws Exception { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = null; - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build(); - cluster.waitActive(); - final FSNamesystem fsn = cluster.getNamesystem(); - - // Set up spys - final FSImage originalImage = fsn.getFSImage(); - NNStorage storage = originalImage.getStorage(); - NNStorage spyStorage = spy(storage); - originalImage.storage = spyStorage; - - final FSEditLog editLog = originalImage.getEditLog(); - FSEditLog spyLog = spy(editLog); - - FSImage spyImage = spy(originalImage); - fsn.dir.fsImage = spyImage; - spyImage.storage.setStorageDirectories( - FSNamesystem.getNamespaceDirs(conf), - FSNamesystem.getNamespaceEditsDirs(conf)); - - // Fail every attempt to open a new edit file - doThrow(new IOException("Injected fault: open")). - when(spyLog).addNewEditLogStream((File)anyObject()); - + File logDir = new File(TEST_DIR, "testFailedOpen"); + logDir.mkdirs(); + FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir); try { - spyLog.close(); - spyLog.open(); - fail("open did not fail even when all directories failed!"); - } catch(IOException ioe) { - LOG.info("Got expected exception", ioe); + logDir.setWritable(false); + log.open(); + fail("Did no throw exception on only having a bad dir"); + } catch (IOException ioe) { + GenericTestUtils.assertExceptionContains( + "no journals successfully started", ioe); } finally { - spyLog.close(); + logDir.setWritable(true); + log.close(); } - - // Reset and try it with a working open - Mockito.reset(spyLog); - spyImage.storage.setStorageDirectories( - FSNamesystem.getNamespaceDirs(conf), - FSNamesystem.getNamespaceEditsDirs(conf)); - spyLog.open(); - - // Close everything off - spyLog.close(); - originalImage.close(); - fsn.close(); } - */ }