hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077643 - in /hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred: JobInProgress.java ResourceEstimator.java
Date Fri, 04 Mar 2011 04:39:39 GMT
Author: omalley
Date: Fri Mar  4 04:39:39 2011
New Revision: 1077643

URL: http://svn.apache.org/viewvc?rev=1077643&view=rev
Log:
commit 6ae4bf9e0e092235ef0f5d9bd13ac24a6e6c75a7
Author: Arun C Murthy <acmurthy@apache.org>
Date:   Thu Aug 5 23:54:58 2010 -0700

    MAPREDUCE-1521. Fixed estimation of map outputs to be consistently use the same number
of maps as reduce slow-start to ensure reduces aren't launched too early. Contributed by Mahadev
Konar.

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1077643&r1=1077642&r2=1077643&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Mar  4 04:39:39 2011
@@ -722,7 +722,10 @@ public class JobInProgress {
           (conf.getFloat("mapred.reduce.slowstart.completed.maps", 
                          DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART) * 
            numMapTasks));
-
+    
+    // ... use the same for estimating the total output of all maps
+    resourceEstimator.setThreshhold(completedMapsForReduceSlowstart);
+    
     // create cleanup two cleanup tips, one map and one reduce.
     cleanup = new TaskInProgress[2];
 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java?rev=1077643&r1=1077642&r2=1077643&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java
Fri Mar  4 04:39:39 2011
@@ -39,7 +39,7 @@ class ResourceEstimator {
 
   private int completedMapsUpdates;
   final private JobInProgress job;
-  final private int threshholdToUse;
+  private int threshholdToUse;
 
   public ResourceEstimator(JobInProgress job) {
     this.job = job;
@@ -106,5 +106,14 @@ class ResourceEstimator {
     }
   }
   
-
+  /**
+   * the number of maps after which reduce starts launching
+   * @param numMaps the number of maps after which reduce starts
+   * launching. It acts as the upper bound for the threshhold, so
+   * that we can get right estimates before we reach these number
+   * of maps.
+   */
+  void setThreshhold(int numMaps) {
+    threshholdToUse = Math.min(threshholdToUse, numMaps);
+  }
 }



Mime
View raw message