hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject hadoop git commit: HDFS-8399. Erasure Coding: unit test the behaviour of BlockManager recovery work for the deleted blocks. Contributed by Rakesh R.
Date Tue, 04 Aug 2015 22:23:51 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7285 9312b168e -> 4de48211c


HDFS-8399. Erasure Coding: unit test the behaviour of BlockManager recovery work for the deleted
blocks. Contributed by Rakesh R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4de48211
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4de48211
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4de48211

Branch: refs/heads/HDFS-7285
Commit: 4de48211c6f8424e1a603f395a214ecc92f5cc74
Parents: 9312b16
Author: Zhe Zhang <zhezhang@cloudera.com>
Authored: Tue Aug 4 15:23:39 2015 -0700
Committer: Zhe Zhang <zhezhang@cloudera.com>
Committed: Tue Aug 4 15:23:39 2015 -0700

----------------------------------------------------------------------
 .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt        |  3 +
 .../server/namenode/TestStripedINodeFile.java   | 82 ++++++++++++++++++++
 2 files changed, 85 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de48211/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index f087bb4..8e54c76 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -385,3 +385,6 @@
 
     HDFS-8804. Erasure Coding: use DirectBufferPool in DFSStripedInputStream for
     buffer allocation. (jing9)
+
+    HDFS-8399. Erasure Coding: unit test the behaviour of BlockManager recovery
+    work for the deleted blocks. (Rakesh R via zhz)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4de48211/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index 9cff614..a9b2048 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -19,16 +19,24 @@ package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstructionStriped;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
@@ -199,4 +207,78 @@ public class TestStripedINodeFile {
     // is 9(= 3 + 6). Consumed storage space should be 1024 * 9 = 9216.
     assertEquals(9216, counts.getStorageSpace());
   }
+
+  /**
+   * Test the behavior of striped and contiguous block deletions.
+   */
+  @Test(timeout = 60000)
+  public void testDeleteOp() throws Exception {
+    MiniDFSCluster cluster = null;
+    try {
+      final int len = 1024;
+      final Path parentDir = new Path("/parentDir");
+      final Path zone = new Path(parentDir, "zone");
+      final Path zoneFile = new Path(zone, "zoneFile");
+      final Path contiguousFile = new Path(parentDir, "someFile");
+      final DistributedFileSystem dfs;
+      final Configuration conf = new Configuration();
+      final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS
+          + HdfsConstants.NUM_PARITY_BLOCKS;
+      conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 2);
+
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE)
+          .build();
+      cluster.waitActive();
+
+      FSNamesystem fsn = cluster.getNamesystem();
+      dfs = cluster.getFileSystem();
+      dfs.mkdirs(zone);
+
+      // create erasure zone
+      dfs.createErasureCodingZone(zone, null, 0);
+      DFSTestUtil.createFile(dfs, zoneFile, len, (short) 1, 0xFEED);
+      DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED);
+      final FSDirectory fsd = fsn.getFSDirectory();
+
+      // Case-1: Verify the behavior of striped blocks
+      // Get blocks of striped file
+      INode inodeStriped = fsd.getINode("/parentDir/zone/zoneFile");
+      assertTrue("Failed to get INodeFile for /parentDir/zone/zoneFile",
+          inodeStriped instanceof INodeFile);
+      INodeFile inodeStripedFile = (INodeFile) inodeStriped;
+      BlockInfo[] stripedBlks = inodeStripedFile.getBlocks();
+      for (BlockInfo blockInfo : stripedBlks) {
+        assertFalse("Mistakenly marked the block as deleted!",
+            blockInfo.isDeleted());
+      }
+
+      // delete erasure zone directory
+      dfs.delete(zone, true);
+      for (BlockInfo blockInfo : stripedBlks) {
+        assertTrue("Didn't mark the block as deleted!", blockInfo.isDeleted());
+      }
+
+      // Case-2: Verify the behavior of contiguous blocks
+      // Get blocks of contiguous file
+      INode inode = fsd.getINode("/parentDir/someFile");
+      assertTrue("Failed to get INodeFile for /parentDir/someFile",
+          inode instanceof INodeFile);
+      INodeFile inodeFile = (INodeFile) inode;
+      BlockInfo[] contiguousBlks = inodeFile.getBlocks();
+      for (BlockInfo blockInfo : contiguousBlks) {
+        assertFalse("Mistakenly marked the block as deleted!",
+            blockInfo.isDeleted());
+      }
+
+      // delete parent directory
+      dfs.delete(parentDir, true);
+      for (BlockInfo blockInfo : contiguousBlks) {
+        assertTrue("Didn't mark the block as deleted!", blockInfo.isDeleted());
+      }
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
 }


Mime
View raw message