hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079258 - in /hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop: mapred/JobInProgress.java mapred/UberTask.java mapreduce/JobCounter.java mapreduce/JobCounter.properties
Date Tue, 08 Mar 2011 06:01:10 GMT
Author: omalley
Date: Tue Mar  8 06:01:10 2011
New Revision: 1079258

URL: http://svn.apache.org/viewvc?rev=1079258&view=rev
Log:
commit 83220625a2489cdc52b5256a006f6775b3502be3
Author: Greg Roelofs <roelofs@yahoo-inc.com>
Date:   Tue Feb 1 20:05:35 2011 -0800

    Uber-specific job counters to help with QA automated testing (ArunR request).

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
Tue Mar  8 06:01:10 2011
@@ -1776,7 +1776,8 @@ public class JobInProgress {
     } else {
       ++runningReduceTasks;
       name = TaskType.REDUCE;
-      counter = JobCounter.TOTAL_LAUNCHED_REDUCES;
+      counter = tip.isUberTask()? JobCounter.TOTAL_LAUNCHED_UBERTASKS
+                                : JobCounter.TOTAL_LAUNCHED_REDUCES;
       if (tip.isSpeculating()) {
         speculativeReduceTasks++;
         metrics.speculateReduce(id);
@@ -3408,6 +3409,8 @@ public class JobInProgress {
       if (!tip.isJobCleanupTask() && !tip.isJobSetupTask()) {
         if (tip.isMapTask()) {
           jobCounters.incrCounter(JobCounter.NUM_FAILED_MAPS, 1);
+        } else if (tip.isUberTask()) {
+          jobCounters.incrCounter(JobCounter.NUM_FAILED_UBERTASKS, 1);
         } else {
           jobCounters.incrCounter(JobCounter.NUM_FAILED_REDUCES, 1);
         }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
Tue Mar  8 06:01:10 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapred.Task.TaskReporter;
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.mapreduce.JobCounter;
 import org.apache.hadoop.mapreduce.MRJobConfig;  // JobContext.SKIP_RECORDS
 import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
 import org.apache.hadoop.mapreduce.TaskType;  // MAP, JOB_SETUP, TASK_CLEANUP...
@@ -413,7 +414,12 @@ class UberTask extends Task {
    *                 completed
    */
   private void updateCounters(Task subtask) {
-    getCounters().incrAllCounters(subtask.getCounters());
+    Counters counters = getCounters();
+    if (counters != null) {
+      counters.incrCounter(subtask.isMapTask()?
+          JobCounter.NUM_UBER_SUBMAPS : JobCounter.NUM_UBER_SUBREDUCES, 1);
+      counters.incrAllCounters(subtask.getCounters());
+    }
   }
 
   @Override

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java
Tue Mar  8 06:01:10 2011
@@ -27,8 +27,12 @@ import org.apache.hadoop.classification.
 public enum JobCounter {
   NUM_FAILED_MAPS, 
   NUM_FAILED_REDUCES,
+  NUM_FAILED_UBERTASKS,
   TOTAL_LAUNCHED_MAPS,
   TOTAL_LAUNCHED_REDUCES,
+  TOTAL_LAUNCHED_UBERTASKS,
+  NUM_UBER_SUBMAPS,
+  NUM_UBER_SUBREDUCES,
   OTHER_LOCAL_MAPS,
   DATA_LOCAL_MAPS,
   RACK_LOCAL_MAPS,

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1079258&r1=1079257&r2=1079258&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties
Tue Mar  8 06:01:10 2011
@@ -16,8 +16,12 @@ CounterGroupName=                  Job C
 
 NUM_FAILED_MAPS.name=              Failed map tasks
 NUM_FAILED_REDUCES.name=           Failed reduce tasks
+NUM_FAILED_UBERTASKS.name=         Failed UberTasks
 TOTAL_LAUNCHED_MAPS.name=          Launched map tasks
 TOTAL_LAUNCHED_REDUCES.name=       Launched reduce tasks
+TOTAL_LAUNCHED_UBERTASKS.name=     Launched UberTasks (as reduce tasks)
+NUM_UBER_SUBMAPS.name=             Completed UberTask sub-maps
+NUM_UBER_SUBREDUCES.name=          Completed UberTask sub-reduces
 OTHER_LOCAL_MAPS.name=             Other local map tasks
 DATA_LOCAL_MAPS.name=              Data-local map tasks
 RACK_LOCAL_MAPS.name=              Rack-local map tasks



Mime
View raw message