hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r417274 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/TaskTracker.java src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
Date Mon, 26 Jun 2006 18:37:39 GMT
Author: cutting
Date: Mon Jun 26 11:37:38 2006
New Revision: 417274

URL: http://svn.apache.org/viewvc?rev=417274&view=rev
Log:
HADOOP-305.  Fix tasktracker to ask for new tasks as soon as a task is finished.  Contributed
by Mahadev.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=417274&r1=417273&r2=417274&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 26 11:37:38 2006
@@ -52,8 +52,13 @@
     error.  (omalley via cutting)
 
 13. HADOOP-304.  Improve error message for
-    UnregisterdDatanodeException, adding expected node name.
+    UnregisterdDatanodeException to include expected node name.
    (Konstantin Shvachko via cutting)
+
+14. HADOOP-305.  Fix TaskTracker to ask for new tasks as soon as a
+    task is finished, rather than waiting for the next heartbeat.
+    This improves performance when tasks are short.
+    (Mahadev Konar via cutting)
 
 
 Release 0.3.2 - 2006-06-09

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=417274&r1=417273&r2=417274&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Mon Jun 26 11:37:38
2006
@@ -79,7 +79,7 @@
 
     private int maxCurrentTasks;
     private int failures;
-    
+    private int finishedCount[] = new int[1];
     /**
      * A list of tips that should be cleaned up.
      */
@@ -284,11 +284,16 @@
             long waitTime = HEARTBEAT_INTERVAL - (now - lastHeartbeat);
             if (waitTime > 0) {
                 try {
-                    Thread.sleep(waitTime);
+                    // sleeps for the wait time, wakes up if a task is finished.
+		    synchronized(finishedCount) {
+                        if (finishedCount[0] == 0) {
+			    finishedCount.wait(waitTime);
+                        }
+                        finishedCount[0] = 0;
+                    }
                 } catch (InterruptedException ie) {
-                }
-                continue;
-            }
+               }
+	    }
             lastHeartbeat = now;
 
             //
@@ -856,6 +861,10 @@
         }
         if (tip != null) {
           tip.taskFinished();
+          synchronized(finishedCount) {
+              finishedCount[0]++;
+              finishedCount.notifyAll();
+          }
         } else {
           LOG.warn("Unknown child task finshed: "+taskid+". Ignored.");
         }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java?rev=417274&r1=417273&r2=417274&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java Mon Jun 26
11:37:38 2006
@@ -114,10 +114,12 @@
         int mapCount = 0;
         for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
             TaskStatus ts = (TaskStatus) it.next();
-            if (ts.getIsMap()) {
-                mapCount++;
-            }
-        }
+	    if (ts.getIsMap() &&
+		((ts.getRunState() == TaskStatus.RUNNING)
+                 || (ts.getRunState() == TaskStatus.UNASSIGNED))) {
+		mapCount++;
+	    }
+	}
         return mapCount;
     }
 
@@ -125,7 +127,16 @@
      * Return the current ReduceTask count
      */
     public int countReduceTasks() {
-        return taskReports.size() - countMapTasks();
+	int reduceCount = 0;
+        for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
+            TaskStatus ts = (TaskStatus) it.next();
+            if ((!ts.getIsMap()) &&
+		((ts.getRunState() == TaskStatus.RUNNING)
+		 || (ts.getRunState() == TaskStatus.UNASSIGNED))) {
+                reduceCount++;
+            }
+        }
+        return reduceCount;
     }
 
     /**



Mime
View raw message