hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vinayakum...@apache.org
Subject [15/50] hadoop git commit: HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from StripedBlockProto. Contributed by Yi Liu.
Date Fri, 14 Aug 2015 10:54:29 GMT
HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from StripedBlockProto. Contributed
by Yi Liu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df589aba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df589aba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df589aba

Branch: refs/heads/HDFS-7285-REBASE
Commit: df589aba2ceb95d7548a2510367e8c711eb7e123
Parents: 476a283
Author: Jing Zhao <jing9@apache.org>
Authored: Fri Jun 12 14:48:53 2015 -0700
Committer: Vinayakumar B <vinayakumarb@apache.org>
Committed: Thu Aug 13 17:09:12 2015 +0530

----------------------------------------------------------------------
 .../src/main/proto/hdfs.proto                   | 10 ---------
 .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt        |  3 +++
 .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 16 --------------
 .../server/namenode/FSImageFormatPBINode.java   | 23 ++++++++++----------
 .../tools/offlineImageViewer/FSImageLoader.java | 20 ++---------------
 .../hadoop-hdfs/src/main/proto/fsimage.proto    |  2 +-
 ...TestOfflineImageViewerWithStripedBlocks.java | 14 +++++-------
 7 files changed, 23 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/df589aba/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index e6db596..dd48d7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -515,16 +515,6 @@ message BlockProto {
 }
 
 /**
- * Striped block information. Besides the basic information for a block,
- * it also contains the number of data/parity blocks.
- */
-message StripedBlockProto {
-  required BlockProto block = 1;
-  optional uint32 dataBlockNum = 2;
-  optional uint32 parityBlockNum = 3;
-}
-
-/**
  * Block and datanodes where is it located
  */
 message BlockWithLocationsProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df589aba/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index fa39d72..2eb8259 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -296,3 +296,6 @@
 
     HDFS-8450. Erasure Coding: Consolidate erasure coding zone related
     implementation into a single class (Rakesh R via vinayakumarb)
+
+    HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from
+    StripedBlockProto. (Yi Liu via jing9)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df589aba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 54e1df9..3da0b97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -183,7 +183,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
 import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
@@ -195,7 +194,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -444,20 +442,6 @@ public class PBHelper {
     return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp());
   }
 
-  public static BlockInfoStriped convert(StripedBlockProto p, ECSchema schema) {
-    return new BlockInfoStriped(convert(p.getBlock()), schema);
-  }
-
-  public static StripedBlockProto convert(BlockInfoStriped blk) {
-    BlockProto bp = BlockProto.newBuilder().setBlockId(blk.getBlockId())
-        .setGenStamp(blk.getGenerationStamp()).setNumBytes(blk.getNumBytes())
-        .build();
-    return StripedBlockProto.newBuilder()
-        .setDataBlockNum(blk.getDataBlockNum())
-        .setParityBlockNum(blk.getParityBlockNum())
-        .setBlock(bp).build();
-  }
-
   public static BlockWithLocationsProto convert(BlockWithLocations blk) {
     BlockWithLocationsProto.Builder builder = BlockWithLocationsProto
         .newBuilder().setBlock(convert(blk.getBlock()))

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df589aba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 69cc865..5cfb707 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@@ -338,11 +337,14 @@ public final class FSImageFormatPBINode {
       short replication = (short) f.getReplication();
       LoaderContext state = parent.getLoaderContext();
 
-      BlockInfo[] blocks = new BlockInfo[bp.size()];
-      for (int i = 0, e = bp.size(); i < e; ++i) {
-        blocks[i] =
-            new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+      BlockInfoContiguous[] blocks = null;
+      if (!f.hasStripedBlocks()) {
+        blocks = new BlockInfoContiguous[bp.size()];
+        for (int i = 0, e = bp.size(); i < e; ++i) {
+          blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+        }
       }
+
       final PermissionStatus permissions = loadPermission(f.getPermission(),
           parent.getLoaderContext().getStringTable());
 
@@ -366,10 +368,9 @@ public final class FSImageFormatPBINode {
       if (f.hasStripedBlocks()) {
         // TODO: HDFS-7859
         ECSchema schema = ErasureCodingSchemaManager.getSystemDefaultSchema();
-        StripedBlocksFeature sb = f.getStripedBlocks();
         stripeFeature = file.addStripedBlocksFeature();
-        for (StripedBlockProto sp : sb.getBlocksList()) {
-          stripeFeature.addBlock(PBHelper.convert(sp, schema));
+        for (BlockProto b : bp) {
+          stripeFeature.addBlock(new BlockInfoStriped(PBHelper.convert(b), schema));
         }
       }
 
@@ -667,14 +668,14 @@ public final class FSImageFormatPBINode {
 
       FileWithStripedBlocksFeature sb = n.getStripedBlocksFeature();
       if (sb != null) {
-        StripedBlocksFeature.Builder builder =
-            StripedBlocksFeature.newBuilder();
         BlockInfoStriped[] sblocks = sb.getBlocks();
         if (sblocks != null) {
           for (BlockInfoStriped sblk : sblocks) {
-            builder.addBlocks(PBHelper.convert(sblk));
+            b.addBlocks(PBHelper.convert(sblk));
           }
         }
+        StripedBlocksFeature.Builder builder =
+            StripedBlocksFeature.newBuilder();
         b.setStripedBlocks(builder.build());
       }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df589aba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index 84518ac..4d8eb35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -43,16 +43,13 @@ import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.XAttrHelper;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
 import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
-import org.apache.hadoop.hdfs.util.StripedBlockUtil;
 import org.apache.hadoop.hdfs.web.JsonUtil;
 import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam;
 import org.apache.hadoop.io.IOUtils;
@@ -649,21 +646,8 @@ class FSImageLoader {
 
   static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
     long size = 0;
-    if (f.hasStripedBlocks()) {
-      List<StripedBlockProto> blocksList = f.getStripedBlocks().getBlocksList();
-      // Get total of actual data block size
-      for (StripedBlockProto p : blocksList) {
-        // Total usage by this striped blocks should be the total of data
-        // blocks and parity blocks
-        size += StripedBlockUtil.spaceConsumedByStripedBlock(p.getBlock()
-            .getNumBytes(), p.getDataBlockNum(), p.getParityBlockNum(),
-            HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
-      }
-    } else {
-      for (HdfsProtos.BlockProto p : f.getBlocksList()) {
-        size += p.getNumBytes();
-      }
-
+    for (HdfsProtos.BlockProto p : f.getBlocksList()) {
+      size += p.getNumBytes();
     }
     return size;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df589aba/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 3f3a71e..4d0e2dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -93,7 +93,7 @@ message INodeSection {
   }
 
   message StripedBlocksFeature {
-    repeated StripedBlockProto blocks = 1;
+    // store striped blocks related information
   }
 
   message AclFeatureProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/df589aba/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
index 2a51f99..f4efbcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.apache.hadoop.hdfs.util.StripedBlockUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -136,28 +135,25 @@ public class TestOfflineImageViewerWithStripedBlocks {
     }
     FSImageLoader loader = FSImageLoader.load(orgFsimage.getAbsolutePath());
     String fileStatus = loader.getFileStatus("/eczone/striped");
-    long expectedSpaceConsumed = StripedBlockUtil.spaceConsumedByStripedBlock(
-        bytes.length, HdfsConstants.NUM_DATA_BLOCKS,
-        HdfsConstants.NUM_PARITY_BLOCKS, HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
+    long expectedFileSize = bytes.length;
 
     // Verify space consumed present in BlockInfoStriped
     FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
     INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
     assertTrue("Invalid block size", fileNode.getBlocks().length > 0);
-    long actualSpaceConsumed = 0;
+    long actualFileSize = 0;
     for (BlockInfo blockInfo : fileNode.getBlocks()) {
       assertTrue("Didn't find block striped information",
           blockInfo instanceof BlockInfoStriped);
-      BlockInfoStriped b = (BlockInfoStriped) blockInfo;
-      actualSpaceConsumed += b.spaceConsumed();
+      actualFileSize += blockInfo.getNumBytes();
     }
 
     assertEquals("Wrongly computed file size contains striped blocks",
-        expectedSpaceConsumed, actualSpaceConsumed);
+        expectedFileSize, actualFileSize);
 
     // Verify space consumed present in filestatus
     String EXPECTED_FILE_SIZE = "\"length\":"
-        + String.valueOf(expectedSpaceConsumed);
+        + String.valueOf(expectedFileSize);
     assertTrue(
         "Wrongly computed file size contains striped blocks, file status:"
             + fileStatus + ". Expected file size is : " + EXPECTED_FILE_SIZE,


Mime
View raw message