hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r671219 - in /hadoop/core/trunk: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobConf.java src/mapred/org/apache/hadoop/mapred/MapTask.java src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Date Tue, 24 Jun 2008 15:42:18 GMT
Author: omalley
Date: Tue Jun 24 08:42:17 2008
New Revision: 671219

URL: http://svn.apache.org/viewvc?rev=671219&view=rev
Log:
HADOOP-3595. Remove deprecated methods for mapred.combine.once 
functionality, which was necessary to providing backwards
compatible combiner semantics for 0.18. Contributed by Chris Douglas.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=671219&r1=671218&r2=671219&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Jun 24 08:42:17 2008
@@ -4,6 +4,10 @@
 
   INCOMPATIBLE CHANGES
 
+    HADOOP-3595. Remove deprecated methods for mapred.combine.once 
+    functionality, which was necessary to providing backwards
+    compatible combiner semantics for 0.18. (cdouglas via omalley)
+
   NEW FEATURES
 
   IMPROVEMENTS

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java?rev=671219&r1=671218&r2=671219&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java Tue Jun 24 08:42:17
2008
@@ -799,20 +799,6 @@
   }
   
   /**
-   * If true, ensures the combiner is run once and only once on output from
-   * the map. Otherwise, combiner may be run zero or more times.
-   */
-  @Deprecated
-  public void setCombineOnceOnly(boolean value) {
-    setBoolean("mapred.combine.once", value);
-  }
-
-  @Deprecated
-  public boolean getCombineOnceOnly() {
-    return getBoolean("mapred.combine.once", false);
-  }
-
-  /**
    * Should speculative execution be used for this job? 
    * Defaults to <code>true</code>.
    * 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=671219&r1=671218&r2=671219&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java Tue Jun 24 08:42:17
2008
@@ -31,9 +31,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
-import java.util.NoSuchElementException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -834,30 +832,11 @@
             writer = new IFile.Writer(job, out, keyClass, valClass, codec);
 
             if (i == partition) {
-              if (job.getCombineOnceOnly()) {
-                Reducer combiner =
-                  (Reducer)ReflectionUtils.newInstance(combinerClass, job);
-                combineCollector.setWriter(writer);
-                combiner.reduce(key, new Iterator<V>() {
-                    private boolean done = false;
-                    public boolean hasNext() { return !done; }
-                    public V next() {
-                      if (done)
-                        throw new NoSuchElementException();
-                      done = true;
-                      return value;
-                    }
-                    public void remove() {
-                      throw new UnsupportedOperationException();
-                    }
-                  }, combineCollector, reporter);
-              } else {
-                final long recordStart = out.getPos();
-                writer.append(key, value);
-                // Note that our map byte count will not be accurate with
-                // compression
-                mapOutputByteCounter.increment(out.getPos() - recordStart);
-              }
+              final long recordStart = out.getPos();
+              writer.append(key, value);
+              // Note that our map byte count will not be accurate with
+              // compression
+              mapOutputByteCounter.increment(out.getPos() - recordStart);
             }
             writer.close();
 
@@ -1057,8 +1036,7 @@
           segmentStart = finalOut.getPos();
           Writer<K, V> writer = 
               new Writer<K, V>(job, finalOut, keyClass, valClass, codec);
-          if (null == combinerClass || job.getCombineOnceOnly() ||
-              numSpills < minSpillsForCombine) {
+          if (null == combinerClass || numSpills < minSpillsForCombine) {
             Merger.writeFile(kvIter, writer, reporter);
           } else {
             combineCollector.setWriter(writer);

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=671219&r1=671218&r2=671219&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Tue Jun 24 08:42:17
2008
@@ -1286,9 +1286,7 @@
       this.numCopiers = conf.getInt("mapred.reduce.parallel.copies", 5);
       this.maxInFlight = 4 * numCopiers;
       this.maxBackoff = conf.getInt("mapred.reduce.copy.backoff", 300);
-      this.combinerClass = conf.getCombineOnceOnly()
-        ? null
-        : conf.getCombinerClass();
+      this.combinerClass = conf.getCombinerClass();
       combineCollector = (null != combinerClass)
         ? new CombineOutputCollector(reduceCombineOutputCounter)
         : null;



Mime
View raw message