Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8B3A210E7E for ; Tue, 1 Sep 2015 01:36:45 +0000 (UTC) Received: (qmail 22699 invoked by uid 500); 1 Sep 2015 01:36:45 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 22631 invoked by uid 500); 1 Sep 2015 01:36:45 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 22622 invoked by uid 99); 1 Sep 2015 01:36:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Sep 2015 01:36:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2CDC9E048E; Tue, 1 Sep 2015 01:36:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinodkv@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-7788. Post-2.6 namenode may not start up with an image containing inodes created with an old release. Contributed by Rushabh Shah. (cherry picked from commit 7ae5255a1613ccfb43646f33eabacf1062c86e93) Date: Tue, 1 Sep 2015 01:36:45 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.6.1 4ec7b6174 -> 02e0b6e30 HDFS-7788. Post-2.6 namenode may not start up with an image containing inodes created with an old release. Contributed by Rushabh Shah. (cherry picked from commit 7ae5255a1613ccfb43646f33eabacf1062c86e93) (cherry picked from commit b9157f92fc3e008e4f3029f8feeaf6acb52eb76f) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/site/resources/image-with-zero-block-size.tar.gz hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java (cherry picked from commit 1faa44d8f4d7b944e99dd0470ea2638c7653a131) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02e0b6e3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02e0b6e3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02e0b6e3 Branch: refs/heads/branch-2.6.1 Commit: 02e0b6e306c63b32a1d0619c52cc68d503b4404f Parents: 4ec7b61 Author: Kihwal Lee Authored: Fri Feb 20 09:09:56 2015 -0600 Committer: Vinod Kumar Vavilapalli Committed: Mon Aug 31 18:32:00 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/namenode/INodeFile.java | 3 ++ .../apache/hadoop/hdfs/util/LongBitFormat.java | 4 ++ .../hdfs/server/namenode/TestFSImage.java | 48 +++++++++++++++++++ .../resources/image-with-zero-block-size.tar.gz | Bin 0 -> 1378 bytes 5 files changed, 58 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e0b6e3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index baeab81..b6cb70b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -73,6 +73,9 @@ Release 2.6.1 - UNRELEASED HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause DataNode to register successfully with only one NameNode.(vinayakumarb) + HDFS-7788. Post-2.6 namenode may not start up with an image containing + inodes created with an old release. (Rushabh Shah via kihwal) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e0b6e3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 5136f8b..1dd6da3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -103,6 +103,9 @@ public class INodeFile extends INodeWithAdditionalFields static long toLong(long preferredBlockSize, short replication, byte storagePolicyID) { long h = 0; + if (preferredBlockSize == 0) { + preferredBlockSize = PREFERRED_BLOCK_SIZE.BITS.getMin(); + } h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h); h = REPLICATION.BITS.combine(replication, h); h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h); http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e0b6e3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java index 863d9f7..9399d84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java @@ -64,4 +64,8 @@ public class LongBitFormat implements Serializable { } return (record & ~MASK) | (value << OFFSET); } + + public long getMin() { + return MIN; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e0b6e3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index f21834e..d19980c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -28,10 +28,13 @@ import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -40,10 +43,14 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; import org.apache.hadoop.hdfs.util.MD5FileUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; import org.junit.Test; public class TestFSImage { + private static final String HADOOP_2_6_ZER0_BLOCK_SIZE_TGZ = + "image-with-zero-block-size.tar.gz"; @Test public void testPersist() throws IOException { Configuration conf = new Configuration(); @@ -183,4 +190,45 @@ public class TestFSImage { } } } + + /** + * In this test case, I have created an image with a file having + * preferredblockSize = 0. We are trying to read this image (since file with + * preferredblockSize = 0 was allowed pre 2.1.0-beta version. The namenode + * after 2.6 version will not be able to read this particular file. + * See HDFS-7788 for more information. + * @throws Exception + */ + @Test + public void testZeroBlockSize() throws Exception { + final Configuration conf = new HdfsConfiguration(); + String tarFile = System.getProperty("test.cache.data", "build/test/cache") + + "/" + HADOOP_2_6_ZER0_BLOCK_SIZE_TGZ; + String testDir = PathUtils.getTestDirName(getClass()); + File dfsDir = new File(testDir, "image-with-zero-block-size"); + if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) { + throw new IOException("Could not delete dfs directory '" + dfsDir + "'"); + } + FileUtil.unTar(new File(tarFile), new File(testDir)); + File nameDir = new File(dfsDir, "name"); + GenericTestUtils.assertExists(nameDir); + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + nameDir.getAbsolutePath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .format(false) + .manageDataDfsDirs(false) + .manageNameDfsDirs(false) + .waitSafeMode(false) + .build(); + try { + FileSystem fs = cluster.getFileSystem(); + Path testPath = new Path("/tmp/zeroBlockFile"); + assertTrue("File /tmp/zeroBlockFile doesn't exist ", fs.exists(testPath)); + assertTrue("Name node didn't come up", cluster.isNameNodeUp(0)); + } finally { + cluster.shutdown(); + //Clean up + FileUtil.fullyDelete(dfsDir); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/02e0b6e3/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz new file mode 100644 index 0000000..41f3105 Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz differ