hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r648021 - in /hadoop/core/branches/branch-0.17: CHANGES.txt src/java/org/apache/hadoop/mapred/MapTask.java
Date Mon, 14 Apr 2008 22:00:24 GMT
Author: cdouglas
Date: Mon Apr 14 15:00:22 2008
New Revision: 648021

URL: http://svn.apache.org/viewvc?rev=648021&view=rev
Log:
HADOOP-3229. Report progress when collecting records from the mapper and
the combiner. Contributed by Doug Cutting.


Modified:
    hadoop/core/branches/branch-0.17/CHANGES.txt
    hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java

Modified: hadoop/core/branches/branch-0.17/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/CHANGES.txt?rev=648021&r1=648020&r2=648021&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.17/CHANGES.txt Mon Apr 14 15:00:22 2008
@@ -559,6 +559,9 @@
    HADOOP-3204. Fixes a problem to do with ReduceTask's LocalFSMerger not
    catching Throwable.  (Amar Ramesh Kamat via ddas)
 
+    HADOOP-3229. Report progress when collecting records from the mapper and
+    the combiner. (Doug Cutting via cdouglas)
+
 Release 0.16.3 - 2008-04-16
 
   BUG FIXES

Modified: hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java?rev=648021&r1=648020&r2=648021&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java Mon Apr
14 15:00:22 2008
@@ -263,6 +263,7 @@
     }
 
     public void collect(K key, V value) throws IOException {
+      reporter.progress();
       this.out.write(key, value);
     }
     
@@ -406,13 +407,14 @@
         deflateFilter = null;
       }
       combineCollector = (null != combinerClass)
-        ? new CombineOutputCollector()
+        ? new CombineOutputCollector(reporter)
         : null;
     }
 
     @SuppressWarnings("unchecked")
     public synchronized void collect(Object key, Object value)
         throws IOException {
+      reporter.progress();
       if (key.getClass() != keyClass) {
         throw new IOException("Type mismatch in key from map: expected "
                               + keyClass.getName() + ", recieved "
@@ -1063,12 +1065,17 @@
    * OutputCollector for the combiner.
    */
   private static class CombineOutputCollector implements OutputCollector {
+    private Reporter reporter;
     private SequenceFile.Writer writer;
+    public CombineOutputCollector(Reporter reporter) {
+      this.reporter = reporter;
+    }
     public synchronized void setWriter(SequenceFile.Writer writer) {
       this.writer = writer;
     }
     public synchronized void collect(Object key, Object value)
         throws IOException {
+        reporter.progress();
         writer.append(key, value);
     }
   }



Mime
View raw message