hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r524174 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/mapred/
Date Fri, 30 Mar 2007 16:59:03 GMT
Author: cutting
Date: Fri Mar 30 09:59:01 2007
New Revision: 524174

URL: http://svn.apache.org/viewvc?view=rev&rev=524174
Log:
HADOOP-1146.  Add a counter for reduce input keys and rename the "reduce input records" counter
to be "reduce input groups."  Contributed by David Bowen.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task_Counter.properties

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=524174&r1=524173&r2=524174
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Mar 30 09:59:01 2007
@@ -48,6 +48,10 @@
 14. HADOOP-1153.  Fix HDFS daemons to correctly stop their threads.
     (Konstantin Shvachko via cutting)
 
+15. HADOOP-1146.  Add a counter for reduce input keys and rename the
+    "reduce input records" counter to be "reduce input groups".
+    (David Bowen via cutting)
+
 
 Release 0.12.3 (not yet released)
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java?view=diff&rev=524174&r1=524173&r2=524174
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java Fri Mar 30 09:59:01
2007
@@ -394,6 +394,7 @@
       while (values.more()) {
         combiner.reduce(values.getKey(), values, combineCollector, reporter);
         values.nextKey();
+        reporter.incrCounter(COMBINE_OUTPUT_RECORDS, 1);
       }
     }
     
@@ -524,6 +525,11 @@
               Configuration conf) 
       throws IOException {
         super(in, comparator, keyClass, valClass, umbilical, conf);
+      }
+      
+      public Object next() {
+        reporter.incrCounter(COMBINE_INPUT_RECORDS, 1);
+        return super.next();
       }
     }
   }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java?view=diff&rev=524174&r1=524173&r2=524174
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java Fri Mar 30 09:59:01
2007
@@ -198,17 +198,23 @@
     }
   }
   private class ReduceValuesIterator extends ValuesIterator {
+    private Reporter reporter;
     public ReduceValuesIterator (SequenceFile.Sorter.RawKeyValueIterator in,
                                WritableComparator comparator, Class keyClass,
                                Class valClass, TaskUmbilicalProtocol umbilical,
-                               Configuration conf)
+                               Configuration conf, Reporter reporter)
     throws IOException {
       super(in, comparator, keyClass, valClass, umbilical, conf);
+      this.reporter = reporter;
     }
     public void informReduceProgress() {
       reducePhase.set(super.in.getProgress().get()); // update progress
       reportProgress(super.umbilical);
     }
+    public Object next() {
+      reporter.incrCounter(REDUCE_INPUT_RECORDS, 1);
+      return super.next();
+    }
   }
 
   public void run(JobConf job, final TaskUmbilicalProtocol umbilical)
@@ -306,10 +312,10 @@
       Class keyClass = job.getMapOutputKeyClass();
       Class valClass = job.getMapOutputValueClass();
       ReduceValuesIterator values = new ReduceValuesIterator(rIter, comparator, 
-                                  keyClass, valClass, umbilical, job);
+                                  keyClass, valClass, umbilical, job, reporter);
       values.informReduceProgress();
       while (values.more()) {
-        reporter.incrCounter(REDUCE_INPUT_RECORDS, 1);
+        reporter.incrCounter(REDUCE_INPUT_GROUPS, 1);
         reducer.reduce(values.getKey(), values, collector, reporter);
         values.nextKey();
         values.informReduceProgress();

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java?view=diff&rev=524174&r1=524173&r2=524174
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task.java Fri Mar 30 09:59:01 2007
@@ -41,6 +41,9 @@
     MAP_OUTPUT_RECORDS,
     MAP_INPUT_BYTES, 
     MAP_OUTPUT_BYTES,
+    COMBINE_INPUT_RECORDS,
+    COMBINE_OUTPUT_RECORDS,
+    REDUCE_INPUT_GROUPS,
     REDUCE_INPUT_RECORDS,
     REDUCE_OUTPUT_RECORDS
   }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task_Counter.properties
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task_Counter.properties?view=diff&rev=524174&r1=524173&r2=524174
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task_Counter.properties (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/Task_Counter.properties Fri Mar
30 09:59:01 2007
@@ -6,6 +6,10 @@
 MAP_INPUT_BYTES.name=          Map input bytes
 MAP_OUTPUT_RECORDS.name=       Map output records
 MAP_OUTPUT_BYTES.name=         Map output bytes
+COMBINE_INPUT_RECORDS.name=    Combine input records
+COMBINE_OUTPUT_RECORDS.name=   Combine output records
+REDUCE_INPUT_GROUPS.name=      Reduce input groups
 REDUCE_INPUT_RECORDS.name=     Reduce input records
 REDUCE_OUTPUT_RECORDS.name=    Reduce output records
+
 



Mime
View raw message