hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kih...@apache.org
Subject hadoop git commit: HDFS-8245. Standby namenode doesn't process DELETED_BLOCK if the addblock request is in edit log. Contributed by Rushabh S Shah.
Date Fri, 08 May 2015 21:36:58 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk e8f66c449 -> 2d4ae3d18


HDFS-8245. Standby namenode doesn't process DELETED_BLOCK if the addblock request is in edit
log. Contributed by Rushabh S Shah.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d4ae3d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d4ae3d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d4ae3d1

Branch: refs/heads/trunk
Commit: 2d4ae3d18bc530fa9f81ee616db8af3395705fb9
Parents: e8f66c4
Author: Kihwal Lee <kihwal@apache.org>
Authored: Fri May 8 16:35:27 2015 -0500
Committer: Kihwal Lee <kihwal@apache.org>
Committed: Fri May 8 16:36:41 2015 -0500

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +
 .../server/blockmanagement/BlockManager.java    | 24 ++++-
 .../server/datanode/TestBlockReplacement.java   | 97 ++++++++++++++++++++
 .../hdfs/server/namenode/ha/TestDNFencing.java  |  4 -
 4 files changed, 121 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d4ae3d1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b45caec..fedef79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -793,6 +793,9 @@ Release 2.7.1 - UNRELEASED
     HDFS-7894. Rolling upgrade readiness is not updated in jmx until query
     command is issued. (Brahma Reddy Battula  via kihwal)
 
+    HDFS-8254. Standby namenode doesn't process DELETED_BLOCK if the add block
+    request is in edit log. (Rushabh S Shah via kihwal)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d4ae3d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 87cb63c..9d9a631 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2313,8 +2313,15 @@ public class BlockManager {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Processing previouly queued message " + rbi);
       }
-      processAndHandleReportedBlock(rbi.getStorageInfo(), 
-          rbi.getBlock(), rbi.getReportedState(), null);
+      if (rbi.getReportedState() == null) {
+        // This is a DELETE_BLOCK request
+        DatanodeStorageInfo storageInfo = rbi.getStorageInfo();
+        removeStoredBlock(rbi.getBlock(),
+            storageInfo.getDatanodeDescriptor());
+      } else {
+        processAndHandleReportedBlock(rbi.getStorageInfo(),
+            rbi.getBlock(), rbi.getReportedState(), null);
+      }
     }
   }
   
@@ -3017,6 +3024,17 @@ public class BlockManager {
     }
   }
 
+  private void removeStoredBlock(DatanodeStorageInfo storageInfo, Block block,
+      DatanodeDescriptor node) {
+    if (shouldPostponeBlocksFromFuture &&
+        namesystem.isGenStampInFuture(block)) {
+      queueReportedBlock(storageInfo, block, null,
+          QUEUE_REASON_FUTURE_GENSTAMP);
+      return;
+    }
+    removeStoredBlock(block, node);
+  }
+
   /**
    * Modify (block-->datanode) map. Possibly generate replication tasks, if the
    * removed block is still valid.
@@ -3194,7 +3212,7 @@ public class BlockManager {
     for (ReceivedDeletedBlockInfo rdbi : srdb.getBlocks()) {
       switch (rdbi.getStatus()) {
       case DELETED_BLOCK:
-        removeStoredBlock(rdbi.getBlock(), node);
+        removeStoredBlock(storageInfo, rdbi.getBlock(), node);
         deleted++;
         break;
       case RECEIVED_BLOCK:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d4ae3d1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
index 2c4795a..70c2a4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
@@ -43,6 +43,8 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.client.BlockReportOptions;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -52,8 +54,11 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.util.Time;
 import org.junit.Test;
@@ -343,6 +348,98 @@ public class TestBlockReplacement {
   }
 
   /**
+   * Standby namenode doesn't queue Delete block request when the add block
+   * request is in the edit log which are yet to be read.
+   * @throws Exception
+   */
+  @Test
+  public void testDeletedBlockWhenAddBlockIsInEdit() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    cluster = new MiniDFSCluster.Builder(conf)
+       .nnTopology(MiniDFSNNTopology.simpleHATopology())
+       .numDataNodes(1).build();
+    DFSClient client = null;
+    try {
+      cluster.waitActive();
+      assertEquals("Number of namenodes is not 2", 2,
+          cluster.getNumNameNodes());
+      // Transitioning the namenode 0 to active.
+      cluster.transitionToActive(0);
+      assertTrue("Namenode 0 should be in active state",
+          cluster.getNameNode(0).isActiveState());
+      assertTrue("Namenode 1 should be in standby state",
+          cluster.getNameNode(1).isStandbyState());
+
+      // Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover
+      // to true.
+      DataNodeTestUtils.triggerHeartbeat(cluster.getDataNodes().get(0));
+      FileSystem fs = cluster.getFileSystem(0);
+
+      // Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale
+      // to false.
+      cluster.getDataNodes().get(0).triggerBlockReport(
+          new BlockReportOptions.Factory().setIncremental(false).build());
+
+      Path fileName = new Path("/tmp.txt");
+      // create a file with one block
+      DFSTestUtil.createFile(fs, fileName, 10L, (short)1, 1234L);
+      DFSTestUtil.waitReplication(fs,fileName, (short)1);
+
+      client = new DFSClient(cluster.getFileSystem(0).getUri(), conf);
+      List<LocatedBlock> locatedBlocks = client.getNamenode().
+          getBlockLocations("/tmp.txt", 0, 10L).getLocatedBlocks();
+      assertTrue(locatedBlocks.size() == 1);
+      assertTrue(locatedBlocks.get(0).getLocations().length == 1);
+
+      // add a second datanode to the cluster
+      cluster.startDataNodes(conf, 1, true, null, null, null, null);
+      assertEquals("Number of datanodes should be 2", 2,
+          cluster.getDataNodes().size());
+
+      DataNode dn0 = cluster.getDataNodes().get(0);
+      DataNode dn1 = cluster.getDataNodes().get(1);
+      String activeNNBPId = cluster.getNamesystem(0).getBlockPoolId();
+      DatanodeDescriptor sourceDnDesc = NameNodeAdapter.getDatanode(
+          cluster.getNamesystem(0), dn0.getDNRegistrationForBP(activeNNBPId));
+      DatanodeDescriptor destDnDesc = NameNodeAdapter.getDatanode(
+          cluster.getNamesystem(0), dn1.getDNRegistrationForBP(activeNNBPId));
+
+      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
+
+      LOG.info("replaceBlock:  " + replaceBlock(block,
+          (DatanodeInfo)sourceDnDesc, (DatanodeInfo)sourceDnDesc,
+          (DatanodeInfo)destDnDesc));
+      // Waiting for the FsDatasetAsyncDsikService to delete the block
+      Thread.sleep(3000);
+      // Triggering the incremental block report to report the deleted block to
+      // namnemode
+      cluster.getDataNodes().get(0).triggerBlockReport(
+         new BlockReportOptions.Factory().setIncremental(true).build());
+
+      cluster.transitionToStandby(0);
+      cluster.transitionToActive(1);
+
+      assertTrue("Namenode 1 should be in active state",
+         cluster.getNameNode(1).isActiveState());
+      assertTrue("Namenode 0 should be in standby state",
+         cluster.getNameNode(0).isStandbyState());
+      client.close();
+
+      // Opening a new client for new active  namenode
+      client = new DFSClient(cluster.getFileSystem(1).getUri(), conf);
+      List<LocatedBlock> locatedBlocks1 = client.getNamenode()
+          .getBlockLocations("/tmp.txt", 0, 10L).getLocatedBlocks();
+
+      assertEquals(1, locatedBlocks1.size());
+      assertEquals("The block should be only on 1 datanode ", 1,
+          locatedBlocks1.get(0).getLocations().length);
+    } finally {
+      IOUtils.cleanup(null, client);
+      cluster.shutdown();
+    }
+  }
+
+  /**
    * @param args
    */
   public static void main(String[] args) throws Exception {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d4ae3d1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index 74358bb..42bf46f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -151,10 +151,6 @@ public class TestDNFencing {
     banner("NN2 Metadata immediately after failover");
     doMetasave(nn2);
     
-    // Even though NN2 considers the blocks over-replicated, it should
-    // post-pone the block invalidation because the DNs are still "stale".
-    assertEquals(30, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
-    
     banner("Triggering heartbeats and block reports so that fencing is completed");
     cluster.triggerHeartbeats();
     cluster.triggerBlockReports();


Mime
View raw message