hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject hadoop git commit: HDFS-7788. Post-2.6 namenode may not start up with an image containing inodes created with an old release. Contributed by Rushabh Shah. (cherry picked from commit 7ae5255a1613ccfb43646f33eabacf1062c86e93)
Date Fri, 20 Feb 2015 15:10:19 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5c2c6b00d -> b9157f92f


HDFS-7788. Post-2.6 namenode may not start up with an image containing inodes created with
an old release. Contributed by Rushabh Shah.
(cherry picked from commit 7ae5255a1613ccfb43646f33eabacf1062c86e93)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9157f92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9157f92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9157f92

Branch: refs/heads/branch-2
Commit: b9157f92fc3e008e4f3029f8feeaf6acb52eb76f
Parents: 5c2c6b0
Author: Kihwal Lee <kihwal@apache.org>
Authored: Fri Feb 20 09:09:56 2015 -0600
Committer: Kihwal Lee <kihwal@apache.org>
Committed: Fri Feb 20 09:09:56 2015 -0600

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 ++
 .../hadoop/hdfs/server/namenode/INodeFile.java  |   3 ++
 .../apache/hadoop/hdfs/util/LongBitFormat.java  |   4 ++
 .../hdfs/server/namenode/TestFSImage.java       |  48 +++++++++++++++++++
 .../resources/image-with-zero-block-size.tar.gz | Bin 0 -> 1452 bytes
 5 files changed, 58 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9157f92/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 363ddca..cdf0e9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -692,6 +692,9 @@ Release 2.7.0 - UNRELEASED
     HDFS-7808. Remove obsolete -ns options in in DFSHAAdmin.java.
     (Arshad Mohammad via wheat9)
 
+    HDFS-7788. Post-2.6 namenode may not start up with an image containing
+    inodes created with an old release. (Rushabh Shah via kihwal)
+
     BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
 
       HDFS-7720. Quota by Storage Type API, tools and ClientNameNode

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9157f92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index eaf72f0..08e9fba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -107,6 +107,9 @@ public class INodeFile extends INodeWithAdditionalFields
     static long toLong(long preferredBlockSize, short replication,
         byte storagePolicyID) {
       long h = 0;
+      if (preferredBlockSize == 0) {
+        preferredBlockSize = PREFERRED_BLOCK_SIZE.BITS.getMin();
+      }
       h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
       h = REPLICATION.BITS.combine(replication, h);
       h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9157f92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
index 863d9f7..9399d84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
@@ -64,4 +64,8 @@ public class LongBitFormat implements Serializable {
     }
     return (record & ~MASK) | (value << OFFSET);
   }
+  
+  public long getMin() {
+    return MIN;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9157f92/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index c68ae04..f7dad18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -28,10 +28,13 @@ import org.junit.Assert;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSOutputStream;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -40,10 +43,14 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.PathUtils;
 import org.junit.Test;
 
 public class TestFSImage {
 
+  private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ =
+      "image-with-zero-block-size.tar.gz";
   @Test
   public void testPersist() throws IOException {
     Configuration conf = new Configuration();
@@ -183,4 +190,45 @@ public class TestFSImage {
       }
     }
   }
+  
+  /**
+   * In this test case, I have created an image with a file having
+   * preferredblockSize = 0. We are trying to read this image (since file with
+   * preferredblockSize = 0 was allowed pre 2.1.0-beta version. The namenode 
+   * after 2.6 version will not be able to read this particular file.
+   * See HDFS-7788 for more information.
+   * @throws Exception
+   */
+  @Test
+  public void testZeroBlockSize() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    String tarFile = System.getProperty("test.cache.data", "build/test/cache")
+      + "/" + HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ;
+    String testDir = PathUtils.getTestDirName(getClass());
+    File dfsDir = new File(testDir, "image-with-zero-block-size");
+    if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) {
+      throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
+    }
+    FileUtil.unTar(new File(tarFile), new File(testDir));
+    File nameDir = new File(dfsDir, "name");
+    GenericTestUtils.assertExists(nameDir);
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, 
+        nameDir.getAbsolutePath());
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+        .format(false)
+        .manageDataDfsDirs(false)
+        .manageNameDfsDirs(false)
+        .waitSafeMode(false)
+        .build();
+    try {
+      FileSystem fs = cluster.getFileSystem();
+      Path testPath = new Path("/tmp/zeroBlockFile");
+      assertTrue("File /tmp/zeroBlockFile doesn't exist ", fs.exists(testPath));
+      assertTrue("Name node didn't come up", cluster.isNameNodeUp(0));
+    } finally {
+      cluster.shutdown();
+      //Clean up
+      FileUtil.fullyDelete(dfsDir);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9157f92/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz
b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz
new file mode 100644
index 0000000..8bfc454
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/image-with-zero-block-size.tar.gz
differ


Mime
View raw message