hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: HDFS-12423. Ozone: TopN container choosing policy should ignore containers that has no pending deletion blocks. Contributed by Yiqun Lin.
Date Wed, 13 Sep 2017 02:59:10 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 830474fe3 -> 9a3159cb6


HDFS-12423. Ozone: TopN container choosing policy should ignore containers that has no pending
deletion blocks. Contributed by Yiqun Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a3159cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a3159cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a3159cb

Branch: refs/heads/HDFS-7240
Commit: 9a3159cb6a25df3b5eb66517ac9b8f028c4a81fb
Parents: 830474f
Author: Weiwei Yang <wwei@apache.org>
Authored: Wed Sep 13 10:50:57 2017 +0800
Committer: Weiwei Yang <wwei@apache.org>
Committed: Wed Sep 13 10:50:57 2017 +0800

----------------------------------------------------------------------
 ...pNOrderedContainerDeletionChoosingPolicy.java | 19 +++++++++++++------
 .../background/BlockDeletingService.java         |  3 +++
 .../common/TestBlockDeletingService.java         |  5 +++++
 .../TestContainerDeletionChoosingPolicy.java     | 12 ++++++++++--
 4 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
index b867179..3f4cdaa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
@@ -66,13 +66,20 @@ public class TopNOrderedContainerDeletionChoosingPolicy
     int currentCount = 0;
     for (ContainerStatus entry : orderedList) {
       if (currentCount < count) {
-        result.add(entry.getContainer());
-        currentCount++;
+        if (entry.getNumPendingDeletionBlocks() > 0) {
+          result.add(entry.getContainer());
+          currentCount++;
 
-        LOG.debug("Select container {} for block deletion, "
-            + "pending deletion blocks num: {}.",
-            entry.getContainer().getContainerName(),
-            entry.getNumPendingDeletionBlocks());
+          LOG.debug(
+              "Select container {} for block deletion, "
+                  + "pending deletion blocks num: {}.",
+              entry.getContainer().getContainerName(),
+              entry.getNumPendingDeletionBlocks());
+        } else {
+          LOG.debug("Stop looking for next container, there is no"
+              + " pending deletion block contained in remaining containers.");
+          break;
+        }
       } else {
         break;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
index 1a2507b..644e928 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
@@ -106,6 +106,9 @@ public class BlockDeletingService extends BackgroundService{
       // configured.
       containers = containerManager.chooseContainerForBlockDeletion(
           containerLimitPerInterval);
+      LOG.info("Plan to choose {} containers for block deletion, "
+          + "actually returns {} valid containers.",
+          containerLimitPerInterval, containers.size());
 
       for(ContainerData container : containers) {
         BlockDeletingTask containerTask =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index ceeb659..7ff5294 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -31,9 +31,11 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
 import org.apache.hadoop.ozone.container.common.helpers.KeyData;
 import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
 import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
+import org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
 import org.apache.hadoop.ozone.container.common.statemachine.background.BlockDeletingService;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.scm.ScmConfigKeys;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
 import org.apache.hadoop.utils.BackgroundService;
@@ -99,6 +101,9 @@ public class TestBlockDeletingService {
 
   private ContainerManager createContainerManager(Configuration conf)
       throws Exception {
+    // use random container choosing policy for testing
+    conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_DELETION_CHOOSING_POLICY,
+        RandomContainerDeletionChoosingPolicy.class.getName());
     conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
         containersDir.getAbsolutePath());
     if (containersDir.exists()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3159cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
index a1c91cc..d6ef9d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
@@ -138,7 +138,8 @@ public class TestContainerDeletionChoosingPolicy {
     int numContainers = 10;
     Random random = new Random();
     Map<String, Integer> name2Count = new HashMap<>();
-    for (int i = 0; i < numContainers; i++) {
+    // create [numContainers + 1] containers
+    for (int i = 0; i <= numContainers; i++) {
       String containerName = OzoneUtils.getRequestID();
       ContainerData data = new ContainerData(containerName);
       containerManager.createContainer(createSingleNodePipeline(containerName),
@@ -146,6 +147,11 @@ public class TestContainerDeletionChoosingPolicy {
       Assert.assertTrue(
           containerManager.getContainerMap().containsKey(containerName));
 
+      // don't create deletion blocks in the last container.
+      if (i == numContainers) {
+        break;
+      }
+
       // create random number of deletion blocks and write to container db
       int deletionBlocks = random.nextInt(numContainers) + 1;
       // record <ContainerName, DeletionCount> value
@@ -170,7 +176,9 @@ public class TestContainerDeletionChoosingPolicy {
     Assert.assertEquals(5, result0.size());
 
     List<ContainerData> result1 = containerManager
-        .chooseContainerForBlockDeletion(numContainers);
+        .chooseContainerForBlockDeletion(numContainers + 1);
+    // the empty deletion blocks container should not be chosen
+    Assert.assertEquals(numContainers, result1.size());
 
     // verify the order of return list
     int lastCount = Integer.MAX_VALUE;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message