hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r778388 - in /hadoop/core/trunk: ./ src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/ src/mapred/org/apache/hadoop/mapred/ src/mapred/org/apache/hadoop/mapreduce/ src/mapred/org/apache/hadoop/mapreduce/lib/input/ src/test...
Date Mon, 25 May 2009 11:22:17 GMT
Author: sharad
Date: Mon May 25 11:22:16 2009
New Revision: 778388

URL: http://svn.apache.org/viewvc?rev=778388&view=rev
Log:
HADOOP-5710. Counter MAP_INPUT_BYTES missing from new mapreduce api. Contributed by Amareshwari
Sriramadasu.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
    hadoop/core/trunk/src/test/findbugsExcludeFile.xml
    hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java
    hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
    hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May 25 11:22:16 2009
@@ -673,6 +673,9 @@
     HADOOP-5252. Streaming overrides -inputformat option (Klaas Bosteels 
     via sharad)
 
+    HADOOP-5710. Counter MAP_INPUT_BYTES missing from new mapreduce api. 
+    (Amareshwari Sriramadasu via sharad)
+
 Release 0.20.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
(original)
+++ hadoop/core/trunk/src/contrib/vaidya/src/java/org/apache/hadoop/vaidya/statistics/job/JobStatistics.java
Mon May 25 11:22:16 2009
@@ -417,8 +417,6 @@
           mapTask.setValue(MapTaskKeys.INPUT_RECORDS, parts[1]);
         } else if (parts[0].equals("Map-Reduce Framework.Map output records")) {
           mapTask.setValue(MapTaskKeys.OUTPUT_RECORDS, parts[1]);
-        } else if (parts[0].equals("Map-Reduce Framework.Map input bytes")) {
-          mapTask.setValue(MapTaskKeys.INPUT_BYTES, parts[1]);
         } else if (parts[0].equals("Map-Reduce Framework.Map output bytes")) {
           mapTask.setValue(MapTaskKeys.OUTPUT_BYTES, parts[1]);
         } else if (parts[0].equals("Map-Reduce Framework.Combine input records")) {
@@ -427,6 +425,8 @@
           mapTask.setValue(MapTaskKeys.COMBINE_OUTPUT_RECORDS, parts[1]);
         } else if (parts[0].equals("Map-Reduce Framework.Spilled Records")) {
           mapTask.setValue(MapTaskKeys.SPILLED_RECORDS, parts[1]);
+        } else if (parts[0].equals("FileInputFormatCounters.BYTES_READ")) {
+          mapTask.setValue(MapTaskKeys.INPUT_BYTES, parts[1]);
         } else {
           System.out.println("MapCounterKey:<"+parts[0]+"> ==> NOT INCLUDED IN PERFORMANCE
ADVISOR MAP TASK");
         }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Counters.java Mon May 25 11:22:16
2009
@@ -36,6 +36,7 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.util.StringUtils;
 
 /**
@@ -398,6 +399,13 @@
    * @return the counter for that name
    */
   public synchronized Counter findCounter(String group, String name) {
+    if (name.equals("MAP_INPUT_BYTES")) {
+      group = FileInputFormat.COUNTER_GROUP; 
+      name = FileInputFormat.BYTES_READ; 
+      LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
+               "Use FileInputFormatCounters as group name and " +
+               " BYTES_READ as counter name instead");
+    }
     return getGroup(group).getCounterForName(name);
   }
 
@@ -411,7 +419,7 @@
    */
   @Deprecated
   public synchronized Counter findCounter(String group, int id, String name) {
-    return getGroup(group).getCounterForName(name);
+    return findCounter(group, name);
   }
 
   /**
@@ -432,7 +440,7 @@
    * @param amount amount by which counter is to be incremented
    */
   public synchronized void incrCounter(String group, String counter, long amount) {
-    getGroup(group).getCounterForName(counter).increment(amount);
+    findCounter(group, counter).increment(amount);
   }
   
   /**

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java Mon May 25 11:22:16
2009
@@ -51,6 +51,7 @@
 import org.apache.hadoop.mapred.IFile.Writer;
 import org.apache.hadoop.mapred.Merger.Segment;
 import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.util.IndexedSortable;
@@ -157,7 +158,9 @@
       throws IOException{
       rawIn = raw;
       inputRecordCounter = reporter.getCounter(TaskCounter.MAP_INPUT_RECORDS);
-      inputByteCounter = reporter.getCounter(TaskCounter.MAP_INPUT_BYTES);
+      inputByteCounter = reporter.getCounter(
+                           FileInputFormat.COUNTER_GROUP,
+                           FileInputFormat.BYTES_READ);
       this.reporter = reporter;
     }
 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.java Mon May 25 11:22:16
2009
@@ -23,7 +23,6 @@
   MAP_INPUT_RECORDS, 
   MAP_OUTPUT_RECORDS,
   MAP_SKIPPED_RECORDS,
-  MAP_INPUT_BYTES, 
   MAP_OUTPUT_BYTES,
   COMBINE_INPUT_RECORDS,
   COMBINE_OUTPUT_RECORDS,

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/TaskCounter.properties Mon May
25 11:22:16 2009
@@ -3,7 +3,6 @@
 CounterGroupName=              Map-Reduce Framework
 
 MAP_INPUT_RECORDS.name=        Map input records
-MAP_INPUT_BYTES.name=          Map input bytes
 MAP_OUTPUT_RECORDS.name=       Map output records
 MAP_OUTPUT_BYTES.name=         Map output bytes
 MAP_SKIPPED_RECORDS.name=      Map skipped records

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
Mon May 25 11:22:16 2009
@@ -49,6 +49,9 @@
  * not split-up and are processed as a whole by {@link Mapper}s.
  */
 public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
+  public static final String COUNTER_GROUP = 
+                                "FileInputFormatCounters";
+  public static final String BYTES_READ = "BYTES_READ";
 
   private static final Log LOG = LogFactory.getLog(FileInputFormat.class);
 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
Mon May 25 11:22:16 2009
@@ -28,9 +28,11 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.MapContext;
 import org.apache.hadoop.util.LineReader;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.Log;
@@ -49,10 +51,13 @@
   private int maxLineLength;
   private LongWritable key = null;
   private Text value = null;
+  private Counter inputByteCounter;
 
   public void initialize(InputSplit genericSplit,
                          TaskAttemptContext context) throws IOException {
     FileSplit split = (FileSplit) genericSplit;
+    inputByteCounter = ((MapContext)context).getCounter(
+      FileInputFormat.COUNTER_GROUP, FileInputFormat.BYTES_READ);
     Configuration job = context.getConfiguration();
     this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength",
                                     Integer.MAX_VALUE);
@@ -101,6 +106,7 @@
         break;
       }
       pos += newSize;
+      inputByteCounter.increment(newSize);
       if (newSize < maxLineLength) {
         break;
       }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
(original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapreduce/lib/input/SequenceFileRecordReader.java
Mon May 25 11:22:16 2009
@@ -25,7 +25,9 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.MapContext;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
@@ -38,12 +40,16 @@
   private K key = null;
   private V value = null;
   protected Configuration conf;
+  private Counter inputByteCounter;
+  private long pos;
   
   @Override
   public void initialize(InputSplit split, 
                          TaskAttemptContext context
                          ) throws IOException, InterruptedException {
     FileSplit fileSplit = (FileSplit) split;
+    inputByteCounter = ((MapContext)context).getCounter(
+      FileInputFormat.COUNTER_GROUP, FileInputFormat.BYTES_READ);
     conf = context.getConfiguration();    
     Path path = fileSplit.getPath();
     FileSystem fs = path.getFileSystem(conf);
@@ -64,7 +70,8 @@
     if (!more) {
       return false;
     }
-    long pos = in.getPosition();
+    inputByteCounter.increment(in.getPosition()-pos);
+    pos = in.getPosition();
     key = (K) in.next(key);
     if (key == null || (pos >= end && in.syncSeen())) {
       more = false;

Modified: hadoop/core/trunk/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/findbugsExcludeFile.xml?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/core/trunk/src/test/findbugsExcludeFile.xml Mon May 25 11:22:16 2009
@@ -84,6 +84,19 @@
        <Bug pattern="BC_UNCONFIRMED_CAST" />
      </Match>
      <!--
+       We intentionally cast from a super class to sub class.
+     -->
+     <Match>
+       <Class name="org.apache.hadoop.mapreduce.lib.input.LineRecordReader" />
+       <Method name="initialize" />
+       <Bug pattern="BC_UNCONFIRMED_CAST" />
+     </Match>
+     <Match>
+       <Class name="org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader" />
+       <Method name="initialize" />
+       <Bug pattern="BC_UNCONFIRMED_CAST" />
+     </Match>
+     <!--
        We intentionally do the get name from the inner class
      -->
      <Match>

Modified: hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java (original)
+++ hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java Mon May 25
11:22:16 2009
@@ -70,7 +70,7 @@
   }
   
   public void testCounters() throws IOException {
-    Enum[] keysWithResource = {TaskCounter.MAP_INPUT_BYTES, 
+    Enum[] keysWithResource = {TaskCounter.MAP_INPUT_RECORDS, 
                                TaskCounter.MAP_OUTPUT_BYTES};
     
     Enum[] keysWithoutResource = {myCounters.TEST1, myCounters.TEST2};

Modified: hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java (original)
+++ hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMiniMRDFSSort.java Mon
May 25 11:22:16 2009
@@ -30,7 +30,6 @@
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
-import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ToolRunner;
@@ -98,8 +97,10 @@
     Sort sort = new Sort();
     assertEquals(ToolRunner.run(job, sort, sortArgs), 0);
     Counters counters = sort.getResult().getCounters();
-    long mapInput = counters.findCounter(TaskCounter.MAP_INPUT_BYTES
-    ).getValue();
+    long mapInput = counters.findCounter(
+      org.apache.hadoop.mapreduce.lib.input.FileInputFormat.COUNTER_GROUP,
+      org.apache.hadoop.mapreduce.lib.input.FileInputFormat.BYTES_READ).
+      getValue();
     long hdfsRead = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP,
                                          "HDFS_BYTES_READ").getValue();
     // the hdfs read should be between 100% and 110% of the map input bytes

Modified: hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java?rev=778388&r1=778387&r2=778388&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
(original)
+++ hadoop/core/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
Mon May 25 11:22:16 2009
@@ -118,6 +118,9 @@
                  out);
     Counters ctrs = job.getCounters();
     System.out.println("Counters: " + ctrs);
+    long mapIn = ctrs.findCounter(FileInputFormat.COUNTER_GROUP, 
+                                  FileInputFormat.BYTES_READ).getValue();
+    assertTrue(mapIn != 0);    
     long combineIn = ctrs.findCounter(COUNTER_GROUP,
                                       "COMBINE_INPUT_RECORDS").getValue();
     long combineOut = ctrs.findCounter(COUNTER_GROUP, 



Mime
View raw message