hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject hadoop git commit: HDFS-9083. Replication violates block placement policy. Contributed by Rushabh Shah.
Date Wed, 28 Oct 2015 19:50:30 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 336be63da -> 13b256ed2


HDFS-9083. Replication violates block placement policy. Contributed by Rushabh Shah.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13b256ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13b256ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13b256ed

Branch: refs/heads/branch-2.7
Commit: 13b256ed2219078653820b544313637ce80a7120
Parents: 336be63
Author: Kihwal Lee <kihwal@apache.org>
Authored: Wed Oct 28 14:49:41 2015 -0500
Committer: Kihwal Lee <kihwal@apache.org>
Committed: Wed Oct 28 14:49:41 2015 -0500

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  2 ++
 .../server/blockmanagement/BlockManager.java    | 10 --------
 .../blockmanagement/TestBlockManager.java       | 24 ++++++++++++++++++++
 3 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b256ed/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e63532d..bd6e9f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -62,6 +62,8 @@ Release 2.7.2 - UNRELEASED
     HDFS-9043. Doc updation for commands in HDFS Federation
     (J.Andreina via vinayakumab)
 
+    HDFS-9083. Replication violates block placement policy (Rushabh Shah)
+
     HDFS-9106. Transfer failure during pipeline recovery causes permanent
     write failures (kihwal)
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b256ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index d770346..c360d4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -224,9 +224,6 @@ public class BlockManager {
 
   final float blocksInvalidateWorkPct;
   final int blocksReplWorkMultiplier;
-
-  /** variable to enable check for enough racks */
-  final boolean shouldCheckForEnoughRacks;
   
   // whether or not to issue block encryption keys.
   final boolean encryptDataTransfer;
@@ -325,9 +322,6 @@ public class BlockManager {
         conf.getInt(
             DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY,
             DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_DEFAULT);
-    this.shouldCheckForEnoughRacks =
-        conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY) == null
-            ? false : true;
 
     this.blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
     this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
@@ -351,7 +345,6 @@ public class BlockManager {
     LOG.info("maxReplication             = " + maxReplication);
     LOG.info("minReplication             = " + minReplication);
     LOG.info("maxReplicationStreams      = " + maxReplicationStreams);
-    LOG.info("shouldCheckForEnoughRacks  = " + shouldCheckForEnoughRacks);
     LOG.info("replicationRecheckInterval = " + replicationRecheckInterval);
     LOG.info("encryptDataTransfer        = " + encryptDataTransfer);
     LOG.info("maxNumBlocksToLog          = " + maxNumBlocksToLog);
@@ -3490,9 +3483,6 @@ public class BlockManager {
   }
 
   boolean blockHasEnoughRacks(Block b) {
-    if (!this.shouldCheckForEnoughRacks) {
-      return true;
-    }
     boolean enoughRacks = false;;
     Collection<DatanodeDescriptor> corruptNodes = 
                                   corruptReplicas.getNodes(b);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b256ed/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index fba840e..e026a53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -816,4 +816,28 @@ public class TestBlockManager {
     Assert.assertFalse(BlockManager.useDelHint(true, delHint, null,
         moreThan1Racks, excessTypes));
   }
+  
+  /**
+   * {@link BlockManager#blockHasEnoughRacks(BlockInfo)} should return false
+   * if all the replicas are on the same rack and shouldn't be dependent on
+   * CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY
+   * @throws Exception
+   */
+  @Test
+  public void testAllReplicasOnSameRack() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    conf.unset(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY);
+    fsn = Mockito.mock(FSNamesystem.class);
+    Mockito.doReturn(true).when(fsn).hasWriteLock();
+    Mockito.doReturn(true).when(fsn).hasReadLock();
+    bm = new BlockManager(fsn, conf);
+    // Add nodes on two racks
+    addNodes(nodes);
+    // Added a new block in blocksMap and all the replicas are on the same rack
+    BlockInfoContiguous blockInfo = addBlockOnNodes(1, rackA);
+    // Since the network toppolgy is multi-rack, the blockHasEnoughRacks 
+    // should return false.
+    assertFalse("Replicas for block is not stored on enough racks", 
+        bm.blockHasEnoughRacks(blockInfo));
+  }
 }


Mime
View raw message