Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 79779 invoked from network); 8 Mar 2011 05:53:24 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 8 Mar 2011 05:53:24 -0000 Received: (qmail 48018 invoked by uid 500); 8 Mar 2011 05:53:23 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 47984 invoked by uid 500); 8 Mar 2011 05:53:23 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 47976 invoked by uid 99); 8 Mar 2011 05:53:23 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 05:53:23 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 05:53:21 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 901692388B6C; Tue, 8 Mar 2011 05:52:59 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079183 - in /hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop: mapred/MapTask.java mapreduce/server/tasktracker/TTConfig.java Date: Tue, 08 Mar 2011 05:52:59 -0000 To: mapreduce-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110308055259.901692388B6C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: omalley Date: Tue Mar 8 05:52:59 2011 New Revision: 1079183 URL: http://svn.apache.org/viewvc?rev=1079183&view=rev Log: commit db221627d41f6cb801bbf25da25fa6d822937d28 Author: Mahadev Konar Date: Fri Nov 12 06:29:25 2010 +0000 How to know what input was sent to a mapper? (log the input split where users can see) from Krishna Ramachandran via mahadev) +++ b/YAHOO-CHANGES.txt + How to know what input was sent to a mapper? (log the input + split where users can see) from + + (Krishna Ramachandran via mahadev) + Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1079183&r1=1079182&r2=1079183&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java Tue Mar 8 05:52:59 2011 @@ -58,6 +58,7 @@ import org.apache.hadoop.mapred.Merger.S import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.map.WrappedMapper; +import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; import org.apache.hadoop.mapreduce.split.JobSplit; import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex; @@ -417,6 +418,7 @@ class MapTask extends Task { job.set(JobContext.MAP_INPUT_FILE, fileSplit.getPath().toString()); job.setLong(JobContext.MAP_INPUT_START, fileSplit.getStart()); job.setLong(JobContext.MAP_INPUT_PATH, fileSplit.getLength()); + job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, fileSplit.toString()); } } @@ -621,6 +623,7 @@ class MapTask extends Task { split = getSplitDetails(new Path(splitIndex.getSplitLocation()), splitIndex.getStartOffset()); + job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, split.toString()); org.apache.hadoop.mapreduce.RecordReader input = new NewTrackingRecordReader (inputFormat.createRecordReader(split, taskContext), reporter); @@ -818,6 +821,7 @@ class MapTask extends Task { LOG.info("soft limit at " + softLimit); LOG.info("bufstart = " + bufstart + "; bufvoid = " + bufvoid); LOG.info("kvstart = " + kvstart + "; length = " + maxRec); + LOG.info("split details: " + job.get(TTConfig.TT_MAP_INPUT_SPLITINFO)); } // k/v serialization Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=1079183&r1=1079182&r2=1079183&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java Tue Mar 8 05:52:59 2011 @@ -70,6 +70,8 @@ public interface TTConfig extends MRConf "mapreduce.tasktracker.instrumentation"; public static final String TT_MAP_SLOTS = "mapreduce.tasktracker.map.tasks.maximum"; + public static final String TT_MAP_INPUT_SPLITINFO = + "mapreduce.tasktracker.map.input.splitinfo"; /** * @deprecated Use {@link #TT_RESOURCE_CALCULATOR_PLUGIN} instead */