hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject hadoop git commit: HDFS-10342. BlockManager#createLocatedBlocks should not check corrupt replicas if none are corrupt. Contributed by Kuhu Shukla.
Date Tue, 09 Aug 2016 16:56:33 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 4aba85875 -> b10c93602


HDFS-10342. BlockManager#createLocatedBlocks should not check corrupt replicas if none are
corrupt. Contributed by Kuhu Shukla.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b10c9360
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b10c9360
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b10c9360

Branch: refs/heads/trunk
Commit: b10c936020e2616609dcb3b2126e8c34328c10ca
Parents: 4aba858
Author: Kihwal Lee <kihwal@apache.org>
Authored: Tue Aug 9 11:55:34 2016 -0500
Committer: Kihwal Lee <kihwal@apache.org>
Committed: Tue Aug 9 11:56:21 2016 -0500

----------------------------------------------------------------------
 .../server/blockmanagement/BlockManager.java    | 37 ++++++++++++++------
 .../blockmanagement/TestBlockManager.java       | 19 ++++++++++
 2 files changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10c9360/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 7411a6a..0e7a59b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1050,17 +1050,19 @@ public class BlockManager implements BlockStatsMXBean {
     final byte[] blockIndices = blk.isStriped() ? new byte[numMachines] : null;
     int j = 0, i = 0;
     if (numMachines > 0) {
+      final boolean noCorrupt = (numCorruptReplicas == 0);
       for(DatanodeStorageInfo storage : blocksMap.getStorages(blk)) {
-        final DatanodeDescriptor d = storage.getDatanodeDescriptor();
-        final boolean replicaCorrupt = corruptReplicas.isReplicaCorrupt(blk, d);
-        if ((isCorrupt || (!replicaCorrupt)) &&
-            storage.getState() != State.FAILED) {
-          machines[j++] = storage;
-          // TODO this can be more efficient
-          if (blockIndices != null) {
-            byte index = ((BlockInfoStriped) blk).getStorageBlockIndex(storage);
-            assert index >= 0;
-            blockIndices[i++] = index;
+        if (storage.getState() != State.FAILED) {
+          if (noCorrupt) {
+            machines[j++] = storage;
+            i = setBlockIndices(blk, blockIndices, i, storage);
+          } else {
+            final DatanodeDescriptor d = storage.getDatanodeDescriptor();
+            final boolean replicaCorrupt = isReplicaCorrupt(blk, d);
+            if (isCorrupt || !replicaCorrupt) {
+              machines[j++] = storage;
+              i = setBlockIndices(blk, blockIndices, i, storage);
+            }
           }
         }
       }
@@ -4586,4 +4588,19 @@ public class BlockManager implements BlockStatsMXBean {
   boolean isGenStampInFuture(Block block) {
     return blockIdManager.isGenStampInFuture(block);
   }
+
+  boolean isReplicaCorrupt(BlockInfo blk, DatanodeDescriptor d) {
+    return corruptReplicas.isReplicaCorrupt(blk, d);
+  }
+
+  private int setBlockIndices(BlockInfo blk, byte[] blockIndices, int i,
+                              DatanodeStorageInfo storage) {
+    // TODO this can be more efficient
+    if (blockIndices != null) {
+      byte index = ((BlockInfoStriped)blk).getStorageBlockIndex(storage);
+      assert index >= 0;
+      blockIndices[i++] = index;
+    }
+    return i;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10c9360/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index bbaf91f..535acd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1265,4 +1265,23 @@ public class TestBlockManager {
       file.delete();
     }
   }
+
+  @Test
+  public void testIsReplicaCorruptCall() throws Exception {
+    BlockManager spyBM = spy(bm);
+    List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
+    List<DatanodeDescriptor> origNodes = getNodes(origStorages);
+    BlockInfo blockInfo = addBlockOnNodes(0, origNodes);
+    spyBM.createLocatedBlocks(new BlockInfo[]{blockInfo}, 3L, false, 0L, 3L,
+        false, false, null, null);
+    verify(spyBM, Mockito.atLeast(0)).
+        isReplicaCorrupt(Mockito.any(BlockInfo.class),
+            Mockito.any(DatanodeDescriptor.class));
+    addCorruptBlockOnNodes(0, origNodes);
+    spyBM.createLocatedBlocks(new BlockInfo[]{blockInfo}, 3L, false, 0L, 3L,
+        false, false, null, null);
+    verify(spyBM, Mockito.atLeast(1)).
+        isReplicaCorrupt(Mockito.any(BlockInfo.class),
+            Mockito.any(DatanodeDescriptor.class));
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message