Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 93767 invoked from network); 8 Mar 2011 06:01:35 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 8 Mar 2011 06:01:35 -0000 Received: (qmail 75232 invoked by uid 500); 8 Mar 2011 06:01:35 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 75207 invoked by uid 500); 8 Mar 2011 06:01:35 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 75196 invoked by uid 99); 8 Mar 2011 06:01:35 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 06:01:35 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 06:01:32 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id D507D2388A5F; Tue, 8 Mar 2011 06:01:10 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079258 - in /hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop: mapred/JobInProgress.java mapred/UberTask.java mapreduce/JobCounter.java mapreduce/JobCounter.properties Date: Tue, 08 Mar 2011 06:01:10 -0000 To: mapreduce-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110308060110.D507D2388A5F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: omalley Date: Tue Mar 8 06:01:10 2011 New Revision: 1079258 URL: http://svn.apache.org/viewvc?rev=1079258&view=rev Log: commit 83220625a2489cdc52b5256a006f6775b3502be3 Author: Greg Roelofs Date: Tue Feb 1 20:05:35 2011 -0800 Uber-specific job counters to help with QA automated testing (ArunR request). Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079258&r1=1079257&r2=1079258&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue Mar 8 06:01:10 2011 @@ -1776,7 +1776,8 @@ public class JobInProgress { } else { ++runningReduceTasks; name = TaskType.REDUCE; - counter = JobCounter.TOTAL_LAUNCHED_REDUCES; + counter = tip.isUberTask()? JobCounter.TOTAL_LAUNCHED_UBERTASKS + : JobCounter.TOTAL_LAUNCHED_REDUCES; if (tip.isSpeculating()) { speculativeReduceTasks++; metrics.speculateReduce(id); @@ -3408,6 +3409,8 @@ public class JobInProgress { if (!tip.isJobCleanupTask() && !tip.isJobSetupTask()) { if (tip.isMapTask()) { jobCounters.incrCounter(JobCounter.NUM_FAILED_MAPS, 1); + } else if (tip.isUberTask()) { + jobCounters.incrCounter(JobCounter.NUM_FAILED_UBERTASKS, 1); } else { jobCounters.incrCounter(JobCounter.NUM_FAILED_REDUCES, 1); } Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java?rev=1079258&r1=1079257&r2=1079258&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java Tue Mar 8 06:01:10 2011 @@ -33,6 +33,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapred.Task.TaskReporter; import org.apache.hadoop.mapred.TaskTracker.TaskInProgress; +import org.apache.hadoop.mapreduce.JobCounter; import org.apache.hadoop.mapreduce.MRJobConfig; // JobContext.SKIP_RECORDS import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex; import org.apache.hadoop.mapreduce.TaskType; // MAP, JOB_SETUP, TASK_CLEANUP... @@ -413,7 +414,12 @@ class UberTask extends Task { * completed */ private void updateCounters(Task subtask) { - getCounters().incrAllCounters(subtask.getCounters()); + Counters counters = getCounters(); + if (counters != null) { + counters.incrCounter(subtask.isMapTask()? + JobCounter.NUM_UBER_SUBMAPS : JobCounter.NUM_UBER_SUBREDUCES, 1); + counters.incrAllCounters(subtask.getCounters()); + } } @Override Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1079258&r1=1079257&r2=1079258&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.java Tue Mar 8 06:01:10 2011 @@ -27,8 +27,12 @@ import org.apache.hadoop.classification. public enum JobCounter { NUM_FAILED_MAPS, NUM_FAILED_REDUCES, + NUM_FAILED_UBERTASKS, TOTAL_LAUNCHED_MAPS, TOTAL_LAUNCHED_REDUCES, + TOTAL_LAUNCHED_UBERTASKS, + NUM_UBER_SUBMAPS, + NUM_UBER_SUBREDUCES, OTHER_LOCAL_MAPS, DATA_LOCAL_MAPS, RACK_LOCAL_MAPS, Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1079258&r1=1079257&r2=1079258&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/JobCounter.properties Tue Mar 8 06:01:10 2011 @@ -16,8 +16,12 @@ CounterGroupName= Job C NUM_FAILED_MAPS.name= Failed map tasks NUM_FAILED_REDUCES.name= Failed reduce tasks +NUM_FAILED_UBERTASKS.name= Failed UberTasks TOTAL_LAUNCHED_MAPS.name= Launched map tasks TOTAL_LAUNCHED_REDUCES.name= Launched reduce tasks +TOTAL_LAUNCHED_UBERTASKS.name= Launched UberTasks (as reduce tasks) +NUM_UBER_SUBMAPS.name= Completed UberTask sub-maps +NUM_UBER_SUBREDUCES.name= Completed UberTask sub-reduces OTHER_LOCAL_MAPS.name= Other local map tasks DATA_LOCAL_MAPS.name= Data-local map tasks RACK_LOCAL_MAPS.name= Rack-local map tasks