hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r388254 - in /lucene/hadoop/trunk/src: java/org/apache/hadoop/mapred/JobTracker.java webapps/mapred/jobtracker.jsp
Date Thu, 23 Mar 2006 20:07:58 GMT
Author: cutting
Date: Thu Mar 23 12:07:56 2006
New Revision: 388254

URL: http://svn.apache.org/viewcvs?rev=388254&view=rev
Log:
Fix for HADOOP-98.  Keep more accurate task counts.  Contributed by Owen O'Malley.

Modified:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=388254&r1=388253&r2=388254&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Thu Mar 23 12:07:56
2006
@@ -101,6 +101,7 @@
 
                             // Remove profile from head of queue
                             trackerExpiryQueue.remove(leastRecent);
+                            String trackerName = leastRecent.getTrackerName();
 
                             // Figure out if last-seen time should be updated, or if tracker
is dead
                             TaskTrackerStatus newProfile = (TaskTrackerStatus) taskTrackers.get(leastRecent.getTrackerName());
@@ -110,12 +111,7 @@
                             if (newProfile != null) {
                                 if (now - newProfile.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL)
{
                                     // Remove completely
-
-                                    TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(leastRecent.getTrackerName());
-                                    if (oldStatus != null) {
-                                        totalMaps -= oldStatus.countMapTasks();
-                                        totalReduces -= oldStatus.countReduceTasks();
-                                    }
+                                    updateTaskTrackerStatus(trackerName, null);
                                     lostTaskTracker(leastRecent.getTrackerName());
                                 } else {
                                     // Update time by inserting latest profile
@@ -478,16 +474,42 @@
     // InterTrackerProtocol
     ////////////////////////////////////////////////////
     public void initialize(String taskTrackerName) {
-        if (taskTrackers.get(taskTrackerName) != null) {
-            TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(taskTrackerName);
-            totalMaps -= oldStatus.countMapTasks();
-            totalReduces -= oldStatus.countReduceTasks();
-
-            lostTaskTracker(taskTrackerName);
+      synchronized (taskTrackers) {
+        boolean seenBefore = updateTaskTrackerStatus(taskTrackerName, null);
+        if (seenBefore) {
+          lostTaskTracker(taskTrackerName);
         }
+      }
     }
 
     /**
+     * Update the last recorded status for the given task tracker.
+     * It assumes that the taskTrackers are locked on entry.
+     * @author Owen O'Malley
+     * @param trackerName The name of the tracker
+     * @param status The new status for the task tracker
+     * @return Was an old status found?
+     */
+    private boolean updateTaskTrackerStatus(String trackerName,
+                                            TaskTrackerStatus status) {
+      TaskTrackerStatus oldStatus = 
+        (TaskTrackerStatus) taskTrackers.get(trackerName);
+      if (oldStatus != null) {
+        totalMaps -= oldStatus.countMapTasks();
+        totalReduces -= oldStatus.countReduceTasks();
+        if (status == null) {
+          taskTrackers.remove(trackerName);
+        }
+      }
+      if (status != null) {
+        totalMaps += status.countMapTasks();
+        totalReduces += status.countReduceTasks();
+        taskTrackers.put(trackerName, status);
+      }
+      return oldStatus != null;
+    }
+    
+    /**
      * Process incoming heartbeat messages from the task trackers.
      */
     public synchronized int emitHeartbeat(TaskTrackerStatus trackerStatus, boolean initialContact)
{
@@ -496,26 +518,20 @@
 
         synchronized (taskTrackers) {
             synchronized (trackerExpiryQueue) {
+                boolean seenBefore = updateTaskTrackerStatus(trackerName,
+                                                             trackerStatus);
                 if (initialContact) {
                     // If it's first contact, then clear out any state hanging around
-                    if (taskTrackers.get(trackerName) != null) {
-                        TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(trackerName);
-                        totalMaps -= oldStatus.countMapTasks();
-                        totalReduces -= oldStatus.countReduceTasks();
+                    if (seenBefore) {
                         lostTaskTracker(trackerName);
                     }
                 } else {
                     // If not first contact, there should be some record of the tracker
-                    if (taskTrackers.get(trackerName) == null) {
+                    if (!seenBefore) {
                         return InterTrackerProtocol.UNKNOWN_TASKTRACKER;
                     }
                 }
 
-                // Store latest state.  If first contact, then save current
-                // state in expiry queue
-                totalMaps += trackerStatus.countMapTasks();
-                totalReduces += trackerStatus.countReduceTasks();
-                taskTrackers.put(trackerName, trackerStatus);
                 if (initialContact) {
                     trackerExpiryQueue.add(trackerStatus);
                 }

Modified: lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp?rev=388254&r1=388253&r2=388254&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp Thu Mar 23 12:07:56 2006
@@ -78,6 +78,17 @@
       out.print("</table>\n");
       out.print("</center>\n");
   }
+
+  public void generateSummaryTable(JspWriter out) throws IOException {
+    ClusterStatus status = tracker.getClusterStatus();
+    out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n"+
+              "<tr><th>Maps</th><th>Reduces</th>" + 
+              "<th>Capacity</th><th>Nodes</th></tr>\n");
+    out.print("<tr><td>" + status.getMapTasks() + "</td><td>" +
+              status.getReduceTasks() + "</td><td>" + 
+              status.getMaxTasks() + "</td><td>" +
+              status.getTaskTrackers() + "</td></tr></table>\n");
+  }
 %>
 
 <html>
@@ -88,6 +99,13 @@
 <h1>JobTracker '<%=trackerLabel%>'</h1>
 
 This JobTracker has been up since <%= new Date(tracker.getStartTime())%>.<br>
+<hr>
+<h2>Cluster Summary</h2>
+<center>
+<% 
+   generateSummaryTable(out); 
+%>
+</center>
 <hr>
 
 



Mime
View raw message