hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r415383 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobTracker.java
Date Mon, 19 Jun 2006 18:32:00 GMT
Author: cutting
Date: Mon Jun 19 11:31:59 2006
New Revision: 415383

URL: http://svn.apache.org/viewvc?rev=415383&view=rev
Log:
HADOOP-299.  Fix the tasktracker, permitting multiple jobs to more easily execute at the same
time.  Contributed by Owen.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=415383&r1=415382&r2=415383&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 19 11:31:59 2006
@@ -6,6 +6,10 @@
  1. HADOOP-298.  Improved progress reports for CopyFiles utility, the
     distributed file copier.  (omalley via cutting)
 
+ 2. HADOOP-299.  Fix some problems in the TaskTracker, permitting
+    multiple jobs to more easily execute at the same time.
+    (omalley via cutting)
+
 
 Release 0.3.2 - 2006-06-09
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=415383&r1=415382&r2=415383&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Jun 19 11:31:59
2006
@@ -39,7 +39,7 @@
     static long RETIRE_JOB_CHECK_INTERVAL;
     static float TASK_ALLOC_EPSILON;
     static float PAD_FRACTION;
-    static float MIN_SLOTS_FOR_PADDING;
+    static final int MIN_CLUSTER_SIZE_FOR_PADDING = 3;
 
     /**
      * Used for formatting the id numbers
@@ -405,8 +405,8 @@
         RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60
* 60 * 1000);
         RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60
* 1000);
         TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon",
0.2f);
-        PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
-        MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;
+        PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 
+                                     0.01f);
 
         // This is a directory of temporary submission files.  We delete it
         // on startup, and can delete any files that we're done with
@@ -643,8 +643,6 @@
         //
         // Compute average map and reduce task numbers across pool
         //
-        int avgMaps = 0;
-        int avgReduces = 0;
         int remainingReduceLoad = 0;
         int remainingMapLoad = 0;
         int numTaskTrackers;
@@ -669,8 +667,6 @@
         }
         
         if (numTaskTrackers > 0) {
-          avgMaps = totalMaps / numTaskTrackers;
-          avgReduces = totalReduces / numTaskTrackers;
           avgMapLoad = remainingMapLoad / numTaskTrackers;
           avgReduceLoad = remainingReduceLoad / numTaskTrackers;
         }
@@ -727,11 +723,12 @@
                     // schedule tasks to the hilt.
                     //
                     totalNeededMaps += job.desiredMaps();
-                    double padding = 0;
-                    if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
-                        padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
+                    int padding = 0;
+                    if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) {
+                      padding = Math.min(maxCurrentTasks,
+                                         (int)(totalNeededMaps * PAD_FRACTION));
                     }
-                    if (totalNeededMaps + padding >= totalCapacity) {
+                    if (totalMaps + padding >= totalCapacity) {
                         break;
                     }
                 }
@@ -762,11 +759,13 @@
                     // schedule tasks to the hilt.
                     //
                     totalNeededReduces += job.desiredReduces();
-                    double padding = 0;
-                    if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
-                        padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
+                    int padding = 0;
+                    if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) {
+                        padding = 
+                          Math.min(maxCurrentTasks,
+                                   (int) (totalNeededReduces * PAD_FRACTION));
                     }
-                    if (totalNeededReduces + padding >= totalCapacity) {
+                    if (totalReduces + padding >= totalCapacity) {
                         break;
                     }
                 }



Mime
View raw message