hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [02/50] [abbrv] hadoop git commit: HDFS-5177. blocksScheduled count should be decremented for abandoned blocks (Contributed by Vinayakumar B)
Date Tue, 05 Apr 2016 19:30:35 GMT
HDFS-5177. blocksScheduled count should be decremented for abandoned blocks (Contributed by
Vinayakumar B)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09d63d5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09d63d5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09d63d5a

Branch: refs/heads/HDFS-1312
Commit: 09d63d5a192b5d6b172f94ff6c94da348fd49ea6
Parents: 690d8a3
Author: Vinayakumar B <vinayakumarb@apache.org>
Authored: Wed Mar 30 14:22:00 2016 +0800
Committer: Vinayakumar B <vinayakumarb@apache.org>
Committed: Wed Mar 30 14:22:00 2016 +0800

----------------------------------------------------------------------
 .../blockmanagement/DatanodeStorageInfo.java    | 10 ++++
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |  4 ++
 .../hadoop/hdfs/TestBlocksScheduledCounter.java | 51 ++++++++++++++++++++
 3 files changed, 65 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d63d5a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index c4729ea..843a8d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -298,6 +298,16 @@ public class DatanodeStorageInfo {
     }
   }
 
+  /**
+   * Decrement the number of blocks scheduled for each given storage. This will
+   * be called during abandon block or delete of UC block.
+   */
+  public static void decrementBlocksScheduled(DatanodeStorageInfo... storages) {
+    for (DatanodeStorageInfo s : storages) {
+      s.getDatanodeDescriptor().decrementBlocksScheduled(s.getStorageType());
+    }
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (this == obj) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d63d5a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 41fd869..ab2f0fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -82,6 +82,10 @@ class FSDirWriteFileOp {
     if (uc == null) {
       return false;
     }
+    if (uc.getUnderConstructionFeature() != null) {
+      DatanodeStorageInfo.decrementBlocksScheduled(uc
+          .getUnderConstructionFeature().getExpectedStorageLocations());
+    }
     fsd.getBlockManager().removeBlockFromMap(uc);
 
     if(NameNode.stateChangeLog.isDebugEnabled()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d63d5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
index b943219..1894278 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
@@ -78,4 +78,55 @@ public class TestBlocksScheduledCounter {
     out.close();   
     assertEquals(0, dn.getBlocksScheduled());
   }
+
+  /**
+   * Abandon block should decrement the scheduledBlocks count for the dataNode.
+   */
+  @Test
+  public void testScheduledBlocksCounterShouldDecrementOnAbandonBlock()
+      throws Exception {
+    cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).numDataNodes(
+        2).build();
+
+    cluster.waitActive();
+    fs = cluster.getFileSystem();
+
+    DatanodeManager datanodeManager = cluster.getNamesystem().getBlockManager()
+        .getDatanodeManager();
+    ArrayList<DatanodeDescriptor> dnList = new ArrayList<DatanodeDescriptor>();
+    datanodeManager.fetchDatanodes(dnList, dnList, false);
+    for (DatanodeDescriptor descriptor : dnList) {
+      assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(),
+          0, descriptor.getBlocksScheduled());
+    }
+
+    cluster.getDataNodes().get(0).shutdown();
+    // open a file an write a few bytes:
+    FSDataOutputStream out = fs.create(new Path("/testBlockScheduledCounter"),
+        (short) 2);
+    for (int i = 0; i < 1024; i++) {
+      out.write(i);
+    }
+    // flush to make sure a block is allocated.
+    out.hflush();
+
+    DatanodeDescriptor abandonedDn = datanodeManager.getDatanode(cluster
+        .getDataNodes().get(0).getDatanodeId());
+    assertEquals("for the abandoned dn scheduled counts should be 0", 0,
+        abandonedDn.getBlocksScheduled());
+
+    for (DatanodeDescriptor descriptor : dnList) {
+      if (descriptor.equals(abandonedDn)) {
+        continue;
+      }
+      assertEquals("Blocks scheduled should be 1 for " + descriptor.getName(),
+          1, descriptor.getBlocksScheduled());
+    }
+    // close the file and the counter should go to zero.
+    out.close();
+    for (DatanodeDescriptor descriptor : dnList) {
+      assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(),
+          0, descriptor.getBlocksScheduled());
+    }
+  }
 }


Mime
View raw message