hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r831813 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobTracker.java src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
Date Mon, 02 Nov 2009 07:54:43 GMT
Author: sharad
Date: Mon Nov  2 07:54:43 2009
New Revision: 831813

URL: http://svn.apache.org/viewvc?rev=831813&view=rev
Log:
MAPREDUCE-1153. Fix tasktracker metrics when trackers are decommissioned. Contributed by Sharad
Agarwal.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=831813&r1=831812&r2=831813&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Nov  2 07:54:43 2009
@@ -61,6 +61,9 @@
     to avoid filling up jobtracker logs on a busy cluster.
     (Ravi Gummadi and Hong Tang via yhemanth)
 
+    MAPREDUCE-1153. Fix tasktracker metrics when trackers are decommissioned.
+    (sharad)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=831813&r1=831812&r2=831813&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Nov  2 07:54:43
2009
@@ -420,15 +420,7 @@
               if ((now - newProfile.getLastSeen()) >
                   tasktrackerExpiryInterval) {
                 // Remove completely after marking the tasks as 'KILLED'
-                lostTaskTracker(current);
-                // tracker is lost, and if it is blacklisted, remove 
-                // it from the count of blacklisted trackers in the cluster
-                if (isBlacklisted(trackerName)) {
-                  faultyTrackers.decrBlackListedTrackers(1);
-                }
-                updateTaskTrackerStatus(trackerName, null);
-                statistics.taskTrackerRemoved(trackerName);
-                getInstrumentation().decTrackers(1);
+                removeTracker(current);
                 // remove the mapping from the hosts list
                 String hostname = newProfile.getHost();
                 hostnameToTaskTracker.get(hostname).remove(trackerName);
@@ -443,6 +435,19 @@
     }
   }
 
+  private void removeTracker(TaskTracker tracker) {
+    lostTaskTracker(tracker);
+    String trackerName = tracker.getStatus().getTrackerName();
+    // tracker is lost, and if it is blacklisted, remove 
+    // it from the count of blacklisted trackers in the cluster
+    if (isBlacklisted(trackerName)) {
+      faultyTrackers.decrBlackListedTrackers(1);
+    }
+    updateTaskTrackerStatus(trackerName, null);
+    statistics.taskTrackerRemoved(trackerName);
+    getInstrumentation().decTrackers(1);
+  }
+
   public synchronized void retireJob(JobID jobid, String historyFile) {
     synchronized (jobs) {
       JobInProgress job = jobs.get(jobid);
@@ -3828,9 +3833,7 @@
             for (TaskTracker tracker : trackers) {
               LOG.info("Decommission: Losing tracker " + tracker + 
                        " on host " + host);
-              lostTaskTracker(tracker); // lose the tracker
-              updateTaskTrackerStatus(
-                tracker.getStatus().getTrackerName(), null);
+              removeTracker(tracker);
             }
             trackersDecommissioned += trackers.size();
           }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java?rev=831813&r1=831812&r2=831813&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerInstrumentation.java
Mon Nov  2 07:54:43 2009
@@ -298,6 +298,18 @@
   }
   
   public void testDecomissionedTrackers() throws IOException {
+    // create TaskTrackerStatus and send heartbeats
+    TaskTrackerStatus[] status = new TaskTrackerStatus[trackers.length];
+    status[0] = getTTStatus(trackers[0], new ArrayList<TaskStatus>());
+    status[1] = getTTStatus(trackers[1], new ArrayList<TaskStatus>());
+    status[2] = getTTStatus(trackers[2], new ArrayList<TaskStatus>());
+    for (int i = 0; i< trackers.length; i++) {
+      FakeObjectUtilities.sendHeartBeat(jobTracker, status[i], false,
+          false, trackers[i], responseId);
+    }
+    
+    assertEquals("Mismatch in number of trackers",
+        trackers.length, mi.numTrackers);
     Set<String> dHosts = new HashSet<String>();
     dHosts.add(hosts[1]);
     assertEquals("Mismatch in number of decommissioned trackers",
@@ -305,6 +317,8 @@
     jobTracker.decommissionNodes(dHosts);
     assertEquals("Mismatch in number of decommissioned trackers",
         1, mi.numTrackersDecommissioned);
+    assertEquals("Mismatch in number of trackers",
+        trackers.length - 1, mi.numTrackers);
   }
   
   static class FakeTaskScheduler extends JobQueueTaskScheduler {



Mime
View raw message