hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r771468 - in /hadoop/core/trunk: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobTracker.java src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
Date Mon, 04 May 2009 22:12:46 GMT
Author: cdouglas
Date: Mon May  4 22:12:45 2009
New Revision: 771468

URL: http://svn.apache.org/viewvc?rev=771468&view=rev
Log:
HADOOP-5733. Add map/reduce slot capacity and blacklisted capacity to JobTracker metrics.
Contributed by Sreekanth Ramakrishnan

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May  4 22:12:45 2009
@@ -295,6 +295,9 @@
     HADOOP-5364. Add certificate expiration warning to HsftpFileSystem and HDFS
     proxy. (Zhiyong Zhang via cdouglas)
 
+    HADOOP-5733. Add map/reduce slot capacity and blacklisted capacity to
+    JobTracker metrics. (Sreekanth Ramakrishnan via cdouglas)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java Mon May  4 22:12:45
2009
@@ -606,11 +606,16 @@
       synchronized (taskTrackers) {
         // remove the capacity of trackers on this host
         for (TaskTrackerStatus status : getStatusesOnHost(hostName)) {
-          totalMapTaskCapacity -= status.getMaxMapTasks();
-          totalReduceTaskCapacity -= status.getMaxReduceTasks();
+          int mapSlots = status.getMaxMapTasks();
+          totalMapTaskCapacity -= mapSlots;
+          int reduceSlots = status.getMaxReduceTasks();
+          totalReduceTaskCapacity -= reduceSlots;
+          getInstrumentation().addBlackListedMapSlots(
+              mapSlots);
+          getInstrumentation().addBlackListedReduceSlots(
+              reduceSlots);
         }
-        numBlacklistedTrackers +=
-          uniqueHostsMap.remove(hostName);
+        numBlacklistedTrackers += uniqueHostsMap.remove(hostName);
       }
     }
     
@@ -620,9 +625,13 @@
         int numTrackersOnHost = 0;
         // add the capacity of trackers on the host
         for (TaskTrackerStatus status : getStatusesOnHost(hostName)) {
-          totalMapTaskCapacity += status.getMaxMapTasks();
-          totalReduceTaskCapacity += status.getMaxReduceTasks();
+          int mapSlots = status.getMaxMapTasks();
+          totalMapTaskCapacity += mapSlots;
+          int reduceSlots = status.getMaxReduceTasks();
+          totalReduceTaskCapacity += reduceSlots;
           numTrackersOnHost++;
+          getInstrumentation().decBlackListedMapSlots(mapSlots);
+          getInstrumentation().decBlackListedReduceSlots(reduceSlots);
         }
         uniqueHostsMap.put(hostName,
                            numTrackersOnHost);
@@ -2627,8 +2636,10 @@
       totalMaps -= oldStatus.countMapTasks();
       totalReduces -= oldStatus.countReduceTasks();
       if (!faultyTrackers.isBlacklisted(oldStatus.getHost())) {
-        totalMapTaskCapacity -= oldStatus.getMaxMapTasks();
-        totalReduceTaskCapacity -= oldStatus.getMaxReduceTasks();
+        int mapSlots = oldStatus.getMaxMapTasks();
+        totalMapTaskCapacity -= mapSlots;
+        int reduceSlots = oldStatus.getMaxReduceTasks();
+        totalReduceTaskCapacity -= reduceSlots;
       }
       if (status == null) {
         taskTrackers.remove(trackerName);
@@ -2647,8 +2658,10 @@
       totalMaps += status.countMapTasks();
       totalReduces += status.countReduceTasks();
       if (!faultyTrackers.isBlacklisted(status.getHost())) {
-        totalMapTaskCapacity += status.getMaxMapTasks();
-        totalReduceTaskCapacity += status.getMaxReduceTasks();
+        int mapSlots = status.getMaxMapTasks();
+        totalMapTaskCapacity += mapSlots;
+        int reduceSlots = status.getMaxReduceTasks();
+        totalReduceTaskCapacity += reduceSlots;
       }
       boolean alreadyPresent = false;
       if (taskTrackers.containsKey(trackerName)) {
@@ -2666,6 +2679,8 @@
         uniqueHostsMap.put(status.getHost(), numTaskTrackersInHost);
       }
     }
+    getInstrumentation().setMapSlots(totalMapTaskCapacity);
+    getInstrumentation().setReduceSlots(totalReduceTaskCapacity);
     return oldStatus != null;
   }
     

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerInstrumentation.java Mon
May  4 22:12:45 2009
@@ -66,4 +66,22 @@
   
   public void decWaitingReduces(JobID id, int task)
   { }
+
+  public void setMapSlots(int slots)
+  { }
+
+  public void setReduceSlots(int slots)
+  { }
+
+  public void addBlackListedMapSlots(int slots)
+  { }
+
+  public void decBlackListedMapSlots(int slots)
+  { }
+
+  public void addBlackListedReduceSlots(int slots)
+  { }
+
+  public void decBlackListedReduceSlots(int slots)
+  { }
 }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java?rev=771468&r1=771467&r2=771468&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTrackerMetricsInst.java Mon May
 4 22:12:45 2009
@@ -38,7 +38,13 @@
   private int numJobsCompleted = 0;
   private int numWaitingMaps = 0;
   private int numWaitingReduces = 0;
-  
+
+  //Cluster status fields.
+  private volatile int numMapSlots = 0;
+  private volatile int numReduceSlots = 0;
+  private int numBlackListedMapSlots = 0;
+  private int numBlackListedReduceSlots = 0;
+
   public JobTrackerMetricsInst(JobTracker tracker, JobConf conf) {
     super(tracker, conf);
     String sessionId = conf.getSessionId();
@@ -57,6 +63,11 @@
    */
   public void doUpdates(MetricsContext unused) {
     synchronized (this) {
+      metricsRecord.setMetric("map_slots", numMapSlots);
+      metricsRecord.setMetric("reduce_slots", numReduceSlots);
+      metricsRecord.incrMetric("blacklisted_maps", numBlackListedMapSlots);
+      metricsRecord.incrMetric("blacklisted_reduces",
+          numBlackListedReduceSlots);
       metricsRecord.incrMetric("maps_launched", numMapTasksLaunched);
       metricsRecord.incrMetric("maps_completed", numMapTasksCompleted);
       metricsRecord.incrMetric("maps_failed", numMapTasksFailed);
@@ -78,6 +89,8 @@
       numJobsCompleted = 0;
       numWaitingMaps = 0;
       numWaitingReduces = 0;
+      numBlackListedMapSlots = 0;
+      numBlackListedReduceSlots = 0;
     }
     metricsRecord.update();
 
@@ -151,4 +164,34 @@
   public synchronized void decWaitingReduces(JobID id, int task){
     numWaitingReduces -= task;
   }
+
+  @Override
+  public void setMapSlots(int slots) {
+    numMapSlots = slots;
+  }
+
+  @Override
+  public void setReduceSlots(int slots) {
+    numReduceSlots = slots;
+  }
+
+  @Override
+  public synchronized void addBlackListedMapSlots(int slots){
+    numBlackListedMapSlots += slots;
+  }
+
+  @Override
+  public synchronized void decBlackListedMapSlots(int slots){
+    numBlackListedMapSlots -= slots;
+  }
+
+  @Override
+  public synchronized void addBlackListedReduceSlots(int slots){
+    numBlackListedReduceSlots += slots;
+  }
+
+  @Override
+  public synchronized void decBlackListedReduceSlots(int slots){
+    numBlackListedReduceSlots -= slots;
+  }
 }



Mime
View raw message