hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xg...@apache.org
Subject [20/57] [abbrv] hadoop git commit: HDFS-11121. Add assertions to BlockInfo#addStorage to protect from breaking reportedBlock-blockGroup mapping. Contributed by Takanobu Asanuma.
Date Thu, 26 Jan 2017 21:12:14 GMT
HDFS-11121. Add assertions to BlockInfo#addStorage to protect from breaking reportedBlock-blockGroup
mapping. Contributed by Takanobu Asanuma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/355b907c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/355b907c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/355b907c

Branch: refs/heads/YARN-5734
Commit: 355b907c09dca8cb813a1ae543dce6c1397a4026
Parents: d79c645
Author: Wei-Chiu Chuang <weichiu@apache.org>
Authored: Sat Jan 21 05:56:22 2017 +0800
Committer: Wei-Chiu Chuang <weichiu@apache.org>
Committed: Sat Jan 21 05:56:22 2017 +0800

----------------------------------------------------------------------
 .../blockmanagement/BlockInfoContiguous.java    |  4 ++++
 .../blockmanagement/BlockInfoStriped.java       |  7 +++++++
 .../server/blockmanagement/TestBlockInfo.java   | 20 +++++++++++++++++---
 .../blockmanagement/TestBlockInfoStriped.java   | 17 +++++++++++++++++
 4 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index 088e068..149efc9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockType;
@@ -55,6 +56,9 @@ public class BlockInfoContiguous extends BlockInfo {
 
   @Override
   boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
+    Preconditions.checkArgument(this.getBlockId() == reportedBlock.getBlockId(),
+        "reported blk_%s is different from stored blk_%s",
+        reportedBlock.getBlockId(), this.getBlockId());
     // find the last null node
     int lastNode = ensureCapacity(1);
     setStorageInfo(lastNode, storage);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
index 10f531b..86e42a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockType;
@@ -114,6 +115,12 @@ public class BlockInfoStriped extends BlockInfo {
 
   @Override
   boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
+    Preconditions.checkArgument(BlockIdManager.isStripedBlockID(
+        reportedBlock.getBlockId()), "reportedBlock is not striped");
+    Preconditions.checkArgument(BlockIdManager.convertToStripedID(
+        reportedBlock.getBlockId()) == this.getBlockId(),
+        "reported blk_%s does not belong to the group of stored blk_%s",
+        reportedBlock.getBlockId(), this.getBlockId());
     int blockIndex = BlockIdManager.getBlockIndex(reportedBlock);
     int index = blockIndex;
     DatanodeStorageInfo old = getStorageInfo(index);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index 4e7cf3d..2dfa900 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -23,6 +23,7 @@ import static org.hamcrest.core.Is.is;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.junit.Assert;
@@ -54,7 +55,8 @@ public class TestBlockInfo {
   public void testAddStorage() throws Exception {
     BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
 
-    final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID",
"127.0.0.1");
+    final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
+        "storageID", "127.0.0.1");
 
     boolean added = blockInfo.addStorage(storage, blockInfo);
 
@@ -66,8 +68,10 @@ public class TestBlockInfo {
   public void testReplaceStorage() throws Exception {
 
     // Create two dummy storages.
-    final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1",
"127.0.0.1");
-    final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(),
new DatanodeStorage("storageID2"));
+    final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo(
+        "storageID1", "127.0.0.1");
+    final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(
+        storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
     final int NUM_BLOCKS = 10;
     BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS];
 
@@ -83,4 +87,14 @@ public class TestBlockInfo {
     Assert.assertThat(added, is(false));
     Assert.assertThat(blockInfos[NUM_BLOCKS/2].getStorageInfo(0), is(storage2));
   }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testAddStorageWithDifferentBlock() throws Exception {
+    BlockInfo blockInfo1 = new BlockInfoContiguous(new Block(1000L), (short) 3);
+    BlockInfo blockInfo2 = new BlockInfoContiguous(new Block(1001L), (short) 3);
+
+    final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
+        "storageID", "127.0.0.1");
+    blockInfo1.addStorage(storage, blockInfo2);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/355b907c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
index 255127f..6a55d8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
@@ -220,4 +220,21 @@ public class TestBlockInfoStriped {
     assertEquals(byteBuffer.array().length, byteStream.toByteArray().length);
     assertArrayEquals(byteBuffer.array(), byteStream.toByteArray());
   }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testAddStorageWithReplicatedBlock() {
+    DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
+        "storageID", "127.0.0.1");
+    BlockInfo replica = new BlockInfoContiguous(new Block(1000L), (short) 3);
+    info.addStorage(storage, replica);
+  }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testAddStorageWithDifferentBlockGroup() {
+    DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
+        "storageID", "127.0.0.1");
+    BlockInfo diffGroup = new BlockInfoStriped(new Block(BASE_ID + 100),
+        testECPolicy);
+    info.addStorage(storage, diffGroup);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message