hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sa...@apache.org
Subject svn commit: r1510818 - in /hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/...
Date Mon, 05 Aug 2013 23:45:16 GMT
Author: sandy
Date: Mon Aug  5 23:45:15 2013
New Revision: 1510818

URL: http://svn.apache.org/r1510818
Log:
MAPREDUCE-5399. Unnecessary Configuration instantiation in IFileInputStream slows down merge.
(Stanislav Barton via Sandy Ryza)

Modified:
    hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
    hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
    hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java

Modified: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1510818&r1=1510817&r2=1510818&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt Mon Aug
 5 23:45:15 2013
@@ -445,6 +445,9 @@ Release 2.1.0-beta - 2013-08-06
     MAPREDUCE-5419. TestSlive is getting FileNotFound Exception (Robert Parker
     via jlowe)
 
+    MAPREDUCE-5399. Unnecessary Configuration instantiation in IFileInputStream
+    slows down merge. (Stanislav Barton via Sandy Ryza)
+
   BREAKDOWN OF HADOOP-8562 SUBTASKS
 
     MAPREDUCE-4739. Some MapReduce tests fail to find winutils.

Modified: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java?rev=1510818&r1=1510817&r2=1510818&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
(original)
+++ hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java
Mon Aug  5 23:45:15 2013
@@ -81,6 +81,8 @@ public class BackupStore<K,V> {
   private boolean inReset = false;
   private boolean clearMarkFlag = false;
   private boolean lastSegmentEOF = false;
+  
+  private Configuration conf;
 
   public BackupStore(Configuration conf, TaskAttemptID taskid)
   throws IOException {
@@ -106,6 +108,8 @@ public class BackupStore<K,V> {
     fileCache = new FileCache(conf);
     tid = taskid;
     
+    this.conf = conf;
+    
     LOG.info("Created a new BackupStore with a memory of " + maxSize);
 
   }
@@ -500,7 +504,7 @@ public class BackupStore<K,V> {
       Reader<K, V> reader = 
         new org.apache.hadoop.mapreduce.task.reduce.InMemoryReader<K, V>(null, 
             (org.apache.hadoop.mapred.TaskAttemptID) tid, 
-            dataOut.getData(), 0, usedSize);
+            dataOut.getData(), 0, usedSize, conf);
       Segment<K, V> segment = new Segment<K, V>(reader, false);
       segmentList.add(segment);
       LOG.debug("Added Memory Segment to List. List Size is " + 

Modified: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java?rev=1510818&r1=1510817&r2=1510818&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
(original)
+++ hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
Mon Aug  5 23:45:15 2013
@@ -24,6 +24,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.mapred.IFile.Reader;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -39,11 +40,11 @@ public class InMemoryReader<K, V> extend
   DataInputBuffer memDataIn = new DataInputBuffer();
   private int start;
   private int length;
-
+  
   public InMemoryReader(MergeManagerImpl<K,V> merger, TaskAttemptID taskAttemptId,
-                        byte[] data, int start, int length)
+                        byte[] data, int start, int length, Configuration conf)
   throws IOException {
-    super(null, null, length - start, null, null);
+    super(conf, null, length - start, null, null);
     this.merger = merger;
     this.taskAttemptId = taskAttemptId;
 

Modified: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java?rev=1510818&r1=1510817&r2=1510818&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
(original)
+++ hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
Mon Aug  5 23:45:15 2013
@@ -613,7 +613,7 @@ public class MergeManagerImpl<K, V> impl
       fullSize -= size;
       Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, 
                                                    mo.getMapId(),
-                                                   data, 0, (int)size);
+                                                   data, 0, (int)size, jobConf);
       inMemorySegments.add(new Segment<K,V>(reader, true, 
                                             (mo.isPrimaryMapOutput() ? 
                                             mergedMapOutputsCounter : null)));



Mime
View raw message