Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B2830D172 for ; Mon, 4 Feb 2013 21:30:01 +0000 (UTC) Received: (qmail 80688 invoked by uid 500); 4 Feb 2013 21:30:01 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 80611 invoked by uid 500); 4 Feb 2013 21:30:01 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 80597 invoked by uid 99); 4 Feb 2013 21:30:00 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Feb 2013 21:30:00 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Feb 2013 21:29:59 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 2D4CE2388ABA; Mon, 4 Feb 2013 21:29:40 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1442375 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/common/ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/ Date: Mon, 04 Feb 2013 21:29:39 -0000 To: hdfs-commits@hadoop.apache.org From: atm@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130204212940.2D4CE2388ABA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: atm Date: Mon Feb 4 21:29:39 2013 New Revision: 1442375 URL: http://svn.apache.org/viewvc?rev=1442375&view=rev Log: HDFS-4462. 2NN will fail to checkpoint after an HDFS upgrade from a pre-federation version of HDFS. Contributed by Aaron T. Myers. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Feb 4 21:29:39 2013 @@ -753,6 +753,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4445. All BKJM ledgers are not checked while tailing, So failover will fail. (Vinay via umamahesh) + HDFS-4462. 2NN will fail to checkpoint after an HDFS upgrade from a + pre-federation version of HDFS. (atm) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java Mon Feb 4 21:29:39 2013 @@ -905,7 +905,7 @@ public abstract class Storage extends St props.setProperty("storageType", storageType.toString()); props.setProperty("namespaceID", String.valueOf(namespaceID)); // Set clusterID in version with federation support - if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { + if (versionSupportsFederation()) { props.setProperty("clusterID", clusterID); } props.setProperty("cTime", String.valueOf(cTime)); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java Mon Feb 4 21:29:39 2013 @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.server.common; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import com.google.common.base.Joiner; @@ -77,6 +79,10 @@ public class StorageInfo { namespaceID = from.namespaceID; cTime = from.cTime; } + + public boolean versionSupportsFederation() { + return LayoutVersion.supports(Feature.FEDERATION, layoutVersion); + } @Override public String toString() { Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java Mon Feb 4 21:29:39 2013 @@ -123,6 +123,10 @@ public class CheckpointSignature extends blockpoolID.equals(si.getBlockPoolID()); } + boolean namespaceIdMatches(FSImage si) { + return namespaceID == si.getStorage().namespaceID; + } + void validateStorageInfo(FSImage si) throws IOException { if (!isSameCluster(si) || !storageVersionMatches(si.getStorage())) { Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Mon Feb 4 21:29:39 2013 @@ -587,7 +587,7 @@ public class NNStorage extends Storage i } // Set Block pool ID in version with federation support - if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { + if (versionSupportsFederation()) { String sbpid = props.getProperty("blockpoolID"); setBlockPoolID(sd.getRoot(), sbpid); } @@ -634,7 +634,7 @@ public class NNStorage extends Storage i ) throws IOException { super.setPropertiesFromFields(props, sd); // Set blockpoolID in version with federation support - if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { + if (versionSupportsFederation()) { props.setProperty("blockpoolID", blockpoolID); } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Mon Feb 4 21:29:39 2013 @@ -475,14 +475,20 @@ public class SecondaryNameNode implement // Returns a token that would be used to upload the merged image. CheckpointSignature sig = namenode.rollEditLog(); - if ((checkpointImage.getNamespaceID() == 0) || - (sig.isSameCluster(checkpointImage) && + boolean loadImage = false; + boolean isFreshCheckpointer = (checkpointImage.getNamespaceID() == 0); + boolean isSameCluster = + (dstStorage.versionSupportsFederation() && sig.isSameCluster(checkpointImage)) || + (!dstStorage.versionSupportsFederation() && sig.namespaceIdMatches(checkpointImage)); + if (isFreshCheckpointer || + (isSameCluster && !sig.storageVersionMatches(checkpointImage.getStorage()))) { // if we're a fresh 2NN, or if we're on the same cluster and our storage // needs an upgrade, just take the storage info from the server. dstStorage.setStorageInfo(sig); dstStorage.setClusterID(sig.getClusterID()); dstStorage.setBlockPoolID(sig.getBlockpoolID()); + loadImage = true; } sig.validateStorageInfo(checkpointImage); @@ -492,7 +498,7 @@ public class SecondaryNameNode implement RemoteEditLogManifest manifest = namenode.getEditLogManifest(sig.mostRecentCheckpointTxId + 1); - boolean loadImage = downloadCheckpointFiles( + loadImage |= downloadCheckpointFiles( fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits doMerge(sig, manifest, loadImage, checkpointImage, namesystem); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java Mon Feb 4 21:29:39 2013 @@ -506,7 +506,11 @@ public abstract class FSImageTestUtil { props.load(fis); IOUtils.closeStream(fis); - props.setProperty(key, value); + if (value == null || value.isEmpty()) { + props.remove(key); + } else { + props.setProperty(key, value); + } out = new FileOutputStream(versionFile); props.store(out, null); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java?rev=1442375&r1=1442374&r2=1442375&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java Mon Feb 4 21:29:39 2013 @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import com.google.common.collect.ImmutableMap; + import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import org.junit.Test; import org.junit.Before; @@ -51,7 +54,7 @@ public class TestSecondaryNameNodeUpgrad } } - private void doIt(String param, String val) throws IOException { + private void doIt(Map paramsToCorrupt) throws IOException { MiniDFSCluster cluster = null; FileSystem fs = null; SecondaryNameNode snn = null; @@ -76,8 +79,12 @@ public class TestSecondaryNameNodeUpgrad snn.shutdown(); for (File versionFile : versionFiles) { - System.out.println("Changing '" + param + "' to '" + val + "' in " + versionFile); - FSImageTestUtil.corruptVersionFile(versionFile, param, val); + for (Map.Entry paramToCorrupt : paramsToCorrupt.entrySet()) { + String param = paramToCorrupt.getKey(); + String val = paramToCorrupt.getValue(); + System.out.println("Changing '" + param + "' to '" + val + "' in " + versionFile); + FSImageTestUtil.corruptVersionFile(versionFile, param, val); + } } snn = new SecondaryNameNode(conf); @@ -94,13 +101,19 @@ public class TestSecondaryNameNodeUpgrad @Test public void testUpgradeLayoutVersionSucceeds() throws IOException { - doIt("layoutVersion", "-39"); + doIt(ImmutableMap.of("layoutVersion", "-39")); + } + + @Test + public void testUpgradePreFedSucceeds() throws IOException { + doIt(ImmutableMap.of("layoutVersion", "-19", "clusterID", "", + "blockpoolID", "")); } @Test public void testChangeNsIDFails() throws IOException { try { - doIt("namespaceID", "2"); + doIt(ImmutableMap.of("namespaceID", "2")); Assert.fail("Should throw InconsistentFSStateException"); } catch(IOException e) { GenericTestUtils.assertExceptionContains("Inconsistent checkpoint fields", e);