hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1296898 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ hadoop-yarn/hadoop-yarn-server...
Date Sun, 04 Mar 2012 23:44:57 GMT
Author: acmurthy
Date: Sun Mar  4 23:44:57 2012
New Revision: 1296898

URL: http://svn.apache.org/viewvc?rev=1296898&view=rev
Log:
MAPREDUCE-3897. Fixed computation of maxActiveAppsPerUser for queues by using capacity and
not max-capacity since we are already scaling it by userLimitFactor. Contributed by Eric Payne.

Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1296898&r1=1296897&r2=1296898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sun Mar  4 23:44:57 2012
@@ -253,6 +253,10 @@ Release 0.23.2 - UNRELEASED
     MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured
     hostname or IP. (tgraves via acmurthy) 
 
+    MAPREDUCE-3897. Fixed computation of maxActiveAppsPerUser for queues by
+    using capacity and not max-capacity since we are already scaling it by
+    userLimitFactor. (Eric Payne via acmurthy) 
+
 Release 0.23.1 - 2012-02-17
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1296898&r1=1296897&r2=1296898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Sun Mar  4 23:44:57 2012
@@ -88,7 +88,8 @@ public class LeafQueue implements CSQueu
   private int maxApplicationsPerUser;
   
   private float maxAMResourcePercent;
-  private int maxActiveApplications;
+  private int maxActiveApplications; // Based on absolute max capacity
+  private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
   private int maxActiveApplicationsPerUser;
   
   private Resource usedResources = Resources.createResource(0);
@@ -167,8 +168,12 @@ public class LeafQueue implements CSQueu
         CSQueueUtils.computeMaxActiveApplications(
             cs.getClusterResources(), this.minimumAllocation,
             maxAMResourcePercent, absoluteMaxCapacity);
+    this.maxActiveAppsUsingAbsCap = 
+            CSQueueUtils.computeMaxActiveApplications(
+                cs.getClusterResources(), this.minimumAllocation,
+                maxAMResourcePercent, absoluteCapacity);
     int maxActiveApplicationsPerUser = 
-        CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit,

+        CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveAppsUsingAbsCap, userLimit,

             userLimitFactor);
 
     this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
@@ -271,6 +276,11 @@ public class LeafQueue implements CSQueu
         "(int)ceil((clusterResourceMemory / minimumAllocation) *" + 
         "maxAMResourcePercent * absoluteMaxCapacity)," + 
         "1) ]" + "\n" +
+        "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
+        " [= max(" + 
+        "(int)ceil((clusterResourceMemory / minimumAllocation) *" + 
+        "maxAMResourcePercent * absoluteCapacity)," + 
+        "1) ]" + "\n" +
         "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser +
         " [= max(" +
         "(int)(maxActiveApplications * (userLimit / 100.0f) * " +
@@ -1376,9 +1386,13 @@ public class LeafQueue implements CSQueu
         CSQueueUtils.computeMaxActiveApplications(
             clusterResource, minimumAllocation, 
             maxAMResourcePercent, absoluteMaxCapacity);
+    maxActiveAppsUsingAbsCap = 
+            CSQueueUtils.computeMaxActiveApplications(
+                clusterResource, minimumAllocation, 
+                maxAMResourcePercent, absoluteCapacity);
     maxActiveApplicationsPerUser = 
         CSQueueUtils.computeMaxActiveApplicationsPerUser(
-            maxActiveApplications, userLimit, userLimitFactor);
+            maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
     
     // Update metrics
     CSQueueUtils.updateQueueStatistics(

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1296898&r1=1296897&r2=1296898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
Sun Mar  4 23:44:57 2012
@@ -158,9 +158,14 @@ public class TestApplicationLimits {
                    queue.getAbsoluteMaximumCapacity()));
     assertEquals(expectedMaxActiveApps, 
                  queue.getMaximumActiveApplications());
+    int expectedMaxActiveAppsUsingAbsCap = 
+            Math.max(1, 
+                (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
+                       csConf.getMaximumApplicationMasterResourcePercent() *
+                       queue.getAbsoluteCapacity()));
     assertEquals(
         (int)Math.ceil(
-            expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) * 
+        		expectedMaxActiveAppsUsingAbsCap * (queue.getUserLimit() / 100.0f) * 
             queue.getUserLimitFactor()), 
         queue.getMaximumActiveApplicationsPerUser());
     assertEquals(
@@ -178,8 +183,13 @@ public class TestApplicationLimits {
                    queue.getAbsoluteMaximumCapacity()));
     assertEquals(expectedMaxActiveApps, 
                  queue.getMaximumActiveApplications());
+    expectedMaxActiveAppsUsingAbsCap = 
+            Math.max(1, 
+                (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
+                       csConf.getMaximumApplicationMasterResourcePercent() *
+                       queue.getAbsoluteCapacity()));
     assertEquals(
-        (int)Math.ceil(expectedMaxActiveApps * 
+        (int)Math.ceil(expectedMaxActiveAppsUsingAbsCap * 
             (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), 
         queue.getMaximumActiveApplicationsPerUser());
     assertEquals(



Mime
View raw message