hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [07/27] hadoop git commit: HDFS-8586. Dead Datanode is allocated for write when client is from deadnode (Contributed by Brahma Reddy Battula)
Date Tue, 30 Jun 2015 19:01:20 GMT
HDFS-8586. Dead Datanode is allocated for write when client is from deadnode (Contributed by
Brahma Reddy Battula)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88ceb382
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88ceb382
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88ceb382

Branch: refs/heads/HADOOP-12111
Commit: 88ceb382ef45bd09cf004cf44aedbabaf3976759
Parents: a95d39f
Author: Vinayakumar B <vinayakumarb@apache.org>
Authored: Mon Jun 29 15:25:03 2015 +0530
Committer: Vinayakumar B <vinayakumarb@apache.org>
Committed: Mon Jun 29 15:25:03 2015 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../BlockPlacementPolicyDefault.java            |  3 +-
 .../hdfs/server/namenode/TestDeadDatanode.java  | 42 ++++++++++++++++++++
 3 files changed, 47 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/88ceb382/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b89e10c..3800184 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -968,6 +968,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8546. Prune cached replicas from DatanodeDescriptor state on replica
     invalidation. (wang)
 
+    HDFS-8586. Dead Datanode is allocated for write when client is from deadnode
+    (Brahma Reddy Battula via vinayakumarb)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/88ceb382/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 21ad01d..9023e0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -454,7 +454,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy
{
       return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
           maxNodesPerRack, results, avoidStaleNodes, storageTypes);
     }
-    if (preferLocalNode && localMachine instanceof DatanodeDescriptor) {
+    if (preferLocalNode && localMachine instanceof DatanodeDescriptor
+        && clusterMap.contains(localMachine)) {
       DatanodeDescriptor localDatanode = (DatanodeDescriptor) localMachine;
       // otherwise try local machine first
       if (excludedNodes.add(localMachine)) { // was not in the excluded list

http://git-wip-us.apache.org/repos/asf/hadoop/blob/88ceb382/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index ff70c3f..c5262d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -18,9 +18,11 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.HashSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,6 +33,9 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -43,6 +48,7 @@ import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
 import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
 import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
 import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.net.Node;
 import org.junit.After;
 import org.junit.Test;
 
@@ -126,4 +132,40 @@ public class TestDeadDatanode {
     assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER
         .getAction());
   }
+
+  @Test
+  public void testDeadNodeAsBlockTarget() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
+    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+    cluster.waitActive();
+
+    String poolId = cluster.getNamesystem().getBlockPoolId();
+    // wait for datanode to be marked live
+    DataNode dn = cluster.getDataNodes().get(0);
+    DatanodeRegistration reg = DataNodeTestUtils.getDNRegistrationForBP(cluster
+        .getDataNodes().get(0), poolId);
+    // Get the updated datanode descriptor
+    BlockManager bm = cluster.getNamesystem().getBlockManager();
+    DatanodeManager dm = bm.getDatanodeManager();
+    Node clientNode = dm.getDatanode(reg);
+
+    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), true,
+        20000);
+
+    // Shutdown and wait for datanode to be marked dead
+    dn.shutdown();
+    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), false,
+        20000);
+    // Get the updated datanode descriptor available in DNM
+    // choose the targets, but local node should not get selected as this is not
+    // part of the cluster anymore
+    DatanodeStorageInfo[] results = bm.chooseTarget4NewBlock("/hello", 3,
+        clientNode, new HashSet<Node>(), 256 * 1024 * 1024L, null, (byte) 7);
+    for (DatanodeStorageInfo datanodeStorageInfo : results) {
+      assertFalse("Dead node should not be choosen", datanodeStorageInfo
+          .getDatanodeDescriptor().equals(clientNode));
+    }
+  }
 }


Mime
View raw message