hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sa...@apache.org
Subject svn commit: r1511461 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/IFile.java src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Date Wed, 07 Aug 2013 19:59:06 GMT
Author: sandy
Date: Wed Aug  7 19:59:05 2013
New Revision: 1511461

URL: http://svn.apache.org/r1511461
Log:
MAPREDUCE-5450. Unnecessary Configuration instantiation in IFileInputStream slows down merge.
(Stanislav Barton and Ted Yu via Sandy Ryza)

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/IFile.java
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1511461&r1=1511460&r2=1511461&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Wed Aug  7 19:59:05 2013
@@ -123,6 +123,9 @@ Release 1.3.0 - unreleased
     MAPREDUCE-5367. Local jobs all use same local working directory
     (Sandy Ryza)
 
+    MAPREDUCE-5450. Unnecessary Configuration instantiation in IFileInputStream
+    slows down merge. (Stanislav Barton and Ted Yu via Sandy Ryza)
+
 Release 1.2.1 - 2013.07.15
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/IFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/IFile.java?rev=1511461&r1=1511460&r2=1511461&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/IFile.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/IFile.java Wed Aug
 7 19:59:05 2013
@@ -462,9 +462,9 @@ class IFile {
     TaskAttemptID taskAttemptId;
     
     public InMemoryReader(RamManager ramManager, TaskAttemptID taskAttemptId,
-                          byte[] data, int start, int length)
+                          byte[] data, int start, int length, Configuration conf)
                           throws IOException {
-      super(null, null, length - start, null, null);
+      super(conf, null, length - start, null, null);
       this.ramManager = ramManager;
       this.taskAttemptId = taskAttemptId;
       

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1511461&r1=1511460&r2=1511461&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Wed
Aug  7 19:59:05 2013
@@ -674,7 +674,8 @@ public class ReduceTask extends Task {
   };
 
   public static class ReduceCopier<K, V> implements ShuffleConsumerPlugin, MRConstants
{
-
+    private JobConf conf;
+    
     /** Reference to the umbilical object */
     private TaskUmbilicalProtocol umbilical;
     private TaskReporter reporter;
@@ -1944,7 +1945,7 @@ public class ReduceTask extends Task {
     @Override
     public void init (ShuffleConsumerPlugin.Context context)throws ClassNotFoundException,
IOException {
       
-      JobConf conf = context.getConf();
+      conf = context.getConf();
       this.reporter = context.getReporter();
       this.umbilical = context.getUmbilical();
       this.reduceTask = context.getReduceTask();
@@ -2442,7 +2443,7 @@ public class ReduceTask extends Task {
           fullSize -= mo.data.length;
           Reader<K, V> reader = 
             new InMemoryReader<K, V>(ramManager, mo.mapAttemptId,
-                                     mo.data, 0, mo.data.length);
+                                     mo.data, 0, mo.data.length, conf);
           Segment<K, V> segment = 
             new Segment<K, V>(reader, true);
           inMemorySegments.add(segment);



Mime
View raw message