hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r468127 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobInProgress.java
Date Thu, 26 Oct 2006 20:50:14 GMT
Author: cutting
Date: Thu Oct 26 13:50:13 2006
New Revision: 468127

URL: http://svn.apache.org/viewvc?view=rev&rev=468127
Log:
HADOOP-578.  Failed tasks are no longer placed at the end of the task queue.  Contributed
by Sanjay.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=468127&r1=468126&r2=468127
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Oct 26 13:50:13 2006
@@ -75,6 +75,11 @@
 20. HADOOP-624.  Fix servlet path to stop a Jetty warning on startup.
     (omalley via cutting)
 
+21. HADOOP-578.  Failed tasks are no longer placed at the end of the
+    task queue.  This was originally done to work around other
+    problems that have now been fixed.  Re-executing failed tasks
+    sooner causes buggy jobs to fail faster.  (Sanjay Dahiya via cutting)
+
 
 Release 0.7.2 - 2006-10-18
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=468127&r1=468126&r2=468127
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Thu Oct 26 13:50:13
2006
@@ -56,8 +56,6 @@
     long finishTime;
 
     private JobConf conf;
-    private int firstMapToTry = 0;
-    private int firstReduceToTry = 0;
     boolean tasksInited = false;
 
     private LocalFileSystem localFs;
@@ -330,7 +328,7 @@
       ArrayList mapCache = (ArrayList)hostToMaps.get(tts.getHost());
       double avgProgress = status.mapProgress() / maps.length;
       int target = findNewTask(tts, clusterSize, avgProgress, 
-                                  maps, firstMapToTry, mapCache);
+                                  maps, mapCache);
       if (target == -1) {
         return null;
       }
@@ -359,7 +357,7 @@
 
         double avgProgress = status.reduceProgress() / reduces.length;
         int target = findNewTask(tts, clusterSize, avgProgress, 
-                                    reduces, firstReduceToTry, null);
+                                    reduces, null);
         if (target == -1) {
           return null;
         }
@@ -388,7 +386,6 @@
                             int clusterSize,
                             double avgProgress,
                             TaskInProgress[] tasks,
-                            int firstTaskToTry,
                             List cachedTasks) {
         String taskTracker = tts.getTrackerName();
         //
@@ -419,8 +416,7 @@
         int failedTarget = -1;
         int specTarget = -1;
         for (int i = 0; i < tasks.length; i++) {
-          int realIdx = (i + firstTaskToTry) % tasks.length; 
-          TaskInProgress task = tasks[realIdx];
+          TaskInProgress task = tasks[i];
           if (task.isRunnable()) {
             // if it failed here and we haven't tried every machine, we
             // don't schedule it here.
@@ -433,15 +429,15 @@
               // failed tasks that aren't running can be scheduled as a last
               // resort
               if (!isRunning && failedTarget == -1) {
-                failedTarget = realIdx;
+                failedTarget = i;
               }
             } else {
               if (!isRunning) {
-                LOG.info("Choosing normal task " + tasks[realIdx].getTIPId());
-                return realIdx;
+                LOG.info("Choosing normal task " + tasks[i].getTIPId());
+                return i;
               } else if (specTarget == -1 &&
                          task.hasSpeculativeTask(avgProgress)) {
-                specTarget = realIdx;
+                specTarget = i;
               }
             }
           }
@@ -625,10 +621,8 @@
         // After this, try to assign tasks with the one after this, so that
         // the failed task goes to the end of the list.
         if (tip.isMapTask()) {
-          firstMapToTry = (tip.getIdWithinJob() + 1) % maps.length;
           failedMapTasks++; 
         } else {
-          firstReduceToTry = (tip.getIdWithinJob() + 1) % reduces.length;
           failedReduceTasks++; 
         }
             



Mime
View raw message