hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079183 - in /hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop: mapred/MapTask.java mapreduce/server/tasktracker/TTConfig.java
Date Tue, 08 Mar 2011 05:52:59 GMT
Author: omalley
Date: Tue Mar  8 05:52:59 2011
New Revision: 1079183

URL: http://svn.apache.org/viewvc?rev=1079183&view=rev
Log:
commit db221627d41f6cb801bbf25da25fa6d822937d28
Author: Mahadev Konar <mahadev@apache.org>
Date:   Fri Nov 12 06:29:25 2010 +0000

     How to know what input was sent to a mapper? (log the input split where users can see)
 from  Krishna Ramachandran via mahadev)
    
    +++ b/YAHOO-CHANGES.txt
    +   How to know what input was sent to a mapper? (log the input
    +  split where users can see)  from
    +
    +  (Krishna Ramachandran via mahadev)
    +

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1079183&r1=1079182&r2=1079183&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java Tue
Mar  8 05:52:59 2011
@@ -58,6 +58,7 @@ import org.apache.hadoop.mapred.Merger.S
 import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
+import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
 import org.apache.hadoop.mapreduce.split.JobSplit;
 import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
 import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
@@ -417,6 +418,7 @@ class MapTask extends Task {
       job.set(JobContext.MAP_INPUT_FILE, fileSplit.getPath().toString());
       job.setLong(JobContext.MAP_INPUT_START, fileSplit.getStart());
       job.setLong(JobContext.MAP_INPUT_PATH, fileSplit.getLength());
+      job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, fileSplit.toString());
     }
   }
 
@@ -621,6 +623,7 @@ class MapTask extends Task {
     split = getSplitDetails(new Path(splitIndex.getSplitLocation()),
         splitIndex.getStartOffset());
 
+    job.set(TTConfig.TT_MAP_INPUT_SPLITINFO, split.toString());
     org.apache.hadoop.mapreduce.RecordReader<INKEY,INVALUE> input =
       new NewTrackingRecordReader<INKEY,INVALUE>
           (inputFormat.createRecordReader(split, taskContext), reporter);
@@ -818,6 +821,7 @@ class MapTask extends Task {
         LOG.info("soft limit at " + softLimit);
         LOG.info("bufstart = " + bufstart + "; bufvoid = " + bufvoid);
         LOG.info("kvstart = " + kvstart + "; length = " + maxRec);
+        LOG.info("split details: " + job.get(TTConfig.TT_MAP_INPUT_SPLITINFO));
       }
 
       // k/v serialization

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=1079183&r1=1079182&r2=1079183&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
Tue Mar  8 05:52:59 2011
@@ -70,6 +70,8 @@ public interface TTConfig extends MRConf
     "mapreduce.tasktracker.instrumentation";
   public static final String TT_MAP_SLOTS = 
     "mapreduce.tasktracker.map.tasks.maximum";
+  public static final String TT_MAP_INPUT_SPLITINFO =
+    "mapreduce.tasktracker.map.input.splitinfo";
   /**
    * @deprecated Use {@link #TT_RESOURCE_CALCULATOR_PLUGIN} instead
    */



Mime
View raw message