hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1134315 - in /hadoop/mapreduce/branches/MR-279: CHANGES.txt mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Date Fri, 10 Jun 2011 13:24:30 GMT
Author: vinodkv
Date: Fri Jun 10 13:24:29 2011
New Revision: 1134315

URL: http://svn.apache.org/viewvc?rev=1134315&view=rev
Log:
Fix more rounding off problems in reduce ramp up. Also fix a bug preventing the application
of the cap on reduce ramp-up. (Sharad Agarwal via vinodkv)

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1134315&r1=1134314&r2=1134315&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri Jun 10 13:24:29 2011
@@ -4,6 +4,10 @@ Trunk (unreleased changes)
 
 
     MAPREDUCE-279
+
+    Fix more rounding off problems in reduce ramp up. Also fix a bug preventing
+    the application of the cap on reduce ramp-up. (Sharad Agarwal via vinodkv)
+
     Fix rounding off problem in reduce ramp up. (sharad)
 
     MAPREDUCE-2582. Cleanup JobHistory event generation.(Siddharth Seth via sharad)   

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1134315&r1=1134314&r2=1134315&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Fri Jun 10 13:24:29 2011
@@ -249,7 +249,7 @@ public class RMContainerAllocator extend
       if (availableMemForMap < mapResourceReqt) {
         //preempt for making space for atleast one map
         int premeptionLimit = Math.max(mapResourceReqt - availableMemForMap, 
-            (int) maxReducePreemptionLimit * memLimit);
+            (int) (maxReducePreemptionLimit * memLimit));
         
         int preemptMem = Math.min(scheduledRequests.maps.size() * mapResourceReqt, 
             premeptionLimit);
@@ -294,34 +294,42 @@ public class RMContainerAllocator extend
       completedMapPercent = 1;
     }
     
-    int scheduledMapMem = scheduledRequests.maps.size() * mapResourceReqt;
-    
-    int mapMemLimit = 0;
-    int reduceMemLimit = 0;
+    int netScheduledMapMem = scheduledRequests.maps.size() * mapResourceReqt
+        + assignedRequests.maps.size() * mapResourceReqt;
+
+    int netScheduledReduceMem = scheduledRequests.reduces.size()
+        * reduceResourceReqt + assignedRequests.reduces.size()
+        * reduceResourceReqt;
+
+    int finalMapMemLimit = 0;
+    int finalReduceMemLimit = 0;
     
     // ramp up the reduces based on completed map percentage
-    int memLimit = getMemLimit();
-    reduceMemLimit = Math.min((int)(completedMapPercent * memLimit),
-        (int) maxReduceRampupLimit * memLimit);
-    mapMemLimit = memLimit - reduceMemLimit;
+    int totalMemLimit = getMemLimit();
+    int idealReduceMemLimit = Math.min((int)(completedMapPercent * totalMemLimit),
+        (int) (maxReduceRampupLimit * totalMemLimit));
+    int idealMapMemLimit = totalMemLimit - idealReduceMemLimit;
 
     // check if there aren't enough maps scheduled, give the free map capacity
     // to reduce
-    if (mapMemLimit > scheduledMapMem) {
-      int unusedMapMemLimit = mapMemLimit - scheduledMapMem;
-      reduceMemLimit = reduceMemLimit + unusedMapMemLimit;
+    if (idealMapMemLimit > netScheduledMapMem) {
+      int unusedMapMemLimit = idealMapMemLimit - netScheduledMapMem;
+      finalReduceMemLimit = idealReduceMemLimit + unusedMapMemLimit;
+      finalMapMemLimit = totalMemLimit - finalReduceMemLimit;
+    } else {
+      finalMapMemLimit = idealMapMemLimit;
+      finalReduceMemLimit = idealReduceMemLimit;
     }
     
-    int netReducesMemScheduled = scheduledRequests.reduces.size() * reduceResourceReqt 
-       + assignedRequests.reduces.size() * reduceResourceReqt;
-    
-    LOG.info("completedMapPercent " + completedMapPercent + 
-        " mapMemLimit:" + mapMemLimit +
-        " reduceMemLimit:" + reduceMemLimit + 
-        " scheduledMapMem:" + scheduledMapMem +
-        " netReducesScheduled:" + netReducesMemScheduled);
+    LOG.info("completedMapPercent " + completedMapPercent +
+        " totalMemLimit:" + totalMemLimit +
+        " finalMapMemLimit:" + finalMapMemLimit +
+        " finalReduceMemLimit:" + finalReduceMemLimit + 
+        " netScheduledMapMem:" + netScheduledMapMem +
+        " netScheduledReduceMem:" + netScheduledReduceMem);
     
-    int rampUp =  (reduceMemLimit - netReducesMemScheduled)/reduceResourceReqt;
+    int rampUp = (finalReduceMemLimit - netScheduledReduceMem)
+        / reduceResourceReqt;
     
     if (rampUp > 0) {
       rampUp = Math.min(rampUp, pendingReduces.size());



Mime
View raw message