hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject hadoop git commit: YARN-3733. Fix DominantRC#compare() does not work as expected if cluster resource is empty. (Rohith Sharmaks via wangda)
Date Thu, 04 Jun 2015 17:27:23 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 718dca485 -> e74e4d7bb


YARN-3733. Fix DominantRC#compare() does not work as expected if cluster resource is empty.
(Rohith Sharmaks via wangda)

(cherry picked from commit ebd797c48fe236b404cf3a125ac9d1f7714e291e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e74e4d7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e74e4d7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e74e4d7b

Branch: refs/heads/branch-2.7
Commit: e74e4d7bb9f7799d91e250aab6281cb6d3faf2ac
Parents: 718dca4
Author: Wangda Tan <wangda@apache.org>
Authored: Thu Jun 4 10:22:57 2015 -0700
Committer: Wangda Tan <wangda@apache.org>
Committed: Thu Jun 4 10:26:06 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../resource/DominantResourceCalculator.java    | 15 +++++
 .../capacity/TestCapacityScheduler.java         | 58 +++++++++++++++++++-
 3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e74e4d7b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2ba4286..d18c477 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -148,6 +148,9 @@ Release 2.7.1 - UNRELEASED
     YARN-3585. NodeManager cannot exit on SHUTDOWN event triggered and NM
     recovery is enabled (Rohith Sharmaks via jlowe)
 
+    YARN-3733. Fix DominantRC#compare() does not work as expected if 
+    cluster resource is empty. (Rohith Sharmaks via wangda)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e74e4d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 6f5b40e..2ee95ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -53,6 +53,21 @@ public class DominantResourceCalculator extends ResourceCalculator {
       return 0;
     }
     
+    if (isInvalidDivisor(clusterResource)) {
+      if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs
+          .getVirtualCores())
+          || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() <
rhs
+              .getVirtualCores())) {
+        return 0;
+      } else if (lhs.getMemory() > rhs.getMemory()
+          || lhs.getVirtualCores() > rhs.getVirtualCores()) {
+        return 1;
+      } else if (lhs.getMemory() < rhs.getMemory()
+          || lhs.getVirtualCores() < rhs.getVirtualCores()) {
+        return -1;
+      }
+    }
+
     float l = getResourceAsValue(clusterResource, lhs, true);
     float r = getResourceAsValue(clusterResource, rhs, true);
     

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e74e4d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index d27cc47..e2e8cc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -123,6 +123,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedule
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
@@ -1159,9 +1160,15 @@ public class TestCapacityScheduler {
 
   private MockRM setUpMove() {
     CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    return setUpMove(conf);
+  }
+
+  private MockRM setUpMove(Configuration config) {
+    CapacitySchedulerConfiguration conf =
+        new CapacitySchedulerConfiguration(config);
     setupQueueConfiguration(conf);
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-      ResourceScheduler.class);
+        ResourceScheduler.class);
     MockRM rm = new MockRM(conf);
     rm.start();
     return rm;
@@ -2640,6 +2647,55 @@ public class TestCapacityScheduler {
     Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
   }
 
+  @Test(timeout = 30000)
+  public void testAMLimitUsage() throws Exception {
+
+    CapacitySchedulerConfiguration config =
+        new CapacitySchedulerConfiguration();
+
+    config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+        DefaultResourceCalculator.class.getName());
+    verifyAMLimitForLeafQueue(config);
+
+    config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+        DominantResourceCalculator.class.getName());
+    verifyAMLimitForLeafQueue(config);
+
+  }
+
+  private void verifyAMLimitForLeafQueue(CapacitySchedulerConfiguration config)
+      throws Exception {
+    MockRM rm = setUpMove(config);
+
+    String queueName = "a1";
+    String userName = "user_0";
+    ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+    LeafQueue queueA =
+        (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
+    Resource amResourceLimit = queueA.getAMResourceLimit();
+
+    Resource amResource =
+        Resource.newInstance(amResourceLimit.getMemory() + 1,
+            amResourceLimit.getVirtualCores() + 1);
+
+    rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+
+    rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+
+    // When AM limit is exceeded, 1 applications will be activated.Rest all
+    // applications will be in pending
+    Assert.assertEquals("PendingApplications should be 1", 1,
+        queueA.getNumPendingApplications());
+    Assert.assertEquals("Active applications should be 1", 1,
+        queueA.getNumActiveApplications());
+
+    Assert.assertEquals("User PendingApplications should be 1", 1, queueA
+        .getUser(userName).getPendingApplications());
+    Assert.assertEquals("User Active applications should be 1", 1, queueA
+        .getUser(userName).getActiveApplications());
+    rm.stop();
+  }
+
   private void setMaxAllocMb(Configuration conf, int maxAllocMb) {
     conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
         maxAllocMb);


Mime
View raw message