tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-2850. Tez MergeManager OOM for small Map Outputs (jeagles)
Date Wed, 21 Oct 2015 18:21:10 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 0e3df0404 -> 35bd5ab0b


TEZ-2850. Tez MergeManager OOM for small Map Outputs (jeagles)

(cherry picked from commit a9cfeb914001c381877657ea39b5de5451740050)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/35bd5ab0
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/35bd5ab0
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/35bd5ab0

Branch: refs/heads/branch-0.7
Commit: 35bd5ab0b58fa4bd2b9162c14baa7e94e22ba00c
Parents: 0e3df04
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Wed Oct 21 13:15:28 2015 -0500
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Wed Oct 21 13:20:45 2015 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 .../runtime/library/common/sort/impl/IFile.java | 25 ++++++++++++--------
 2 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/35bd5ab0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fdc5be2..29e61bf 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.7.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES
+  TEZ-2850. Tez MergeManager OOM for small Map Outputs
   TEZ-2886. Ability to merge AM credentials with DAG credentials.
   TEZ-2896. Fix thread names used during Input/Output initialization.
   TEZ-2866. Tez UI: Newly added columns wont be displayed by default in tables
@@ -294,6 +295,7 @@ Release 0.6.3: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2850. Tez MergeManager OOM for small Map Outputs
   TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails
   TEZ-2855. Fix a potential NPE while routing VertexManager events.
   TEZ-2716. DefaultSorter.isRleNeeded not thread safe

http://git-wip-us.apache.org/repos/asf/tez/blob/35bd5ab0/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java
index 8dcbf6d..20f44dd 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java
@@ -501,10 +501,10 @@ public class IFile {
     protected int recNo = 1;
     protected int originalKeyLength;
     protected int prevKeyLength;
-    protected int currentKeyLength;
-    protected int currentValueLength;
     byte keyBytes[] = new byte[0];
 
+    protected int currentKeyLength;
+    protected int currentValueLength;
     long startPos;
 
     /**
@@ -564,21 +564,26 @@ public class IFile {
                   TezCounter readsCounter, TezCounter bytesReadCounter,
                   boolean readAhead, int readAheadLength,
                   int bufferSize, boolean isCompressed) throws IOException {
-      checksumIn = new IFileInputStream(in, length, readAhead, readAheadLength/*, isCompressed*/);
-      if (isCompressed && codec != null) {
-        decompressor = CodecPool.getDecompressor(codec);
-        if (decompressor != null) {
-          this.in = codec.createInputStream(checksumIn, decompressor);
+      if (in != null) {
+        checksumIn = new IFileInputStream(in, length, readAhead,
+            readAheadLength/* , isCompressed */);
+        if (isCompressed && codec != null) {
+          decompressor = CodecPool.getDecompressor(codec);
+          if (decompressor != null) {
+            this.in = codec.createInputStream(checksumIn, decompressor);
+          } else {
+            LOG.warn("Could not obtain decompressor from CodecPool");
+            this.in = checksumIn;
+          }
         } else {
-          LOG.warn("Could not obtain decompressor from CodecPool");
           this.in = checksumIn;
         }
+        startPos = checksumIn.getPosition();
       } else {
-        this.in = checksumIn;
+        this.in = null;
       }
 
       this.dataIn = new DataInputStream(this.in);
-      startPos = checksumIn.getPosition();
       this.readRecordsCounter = readsCounter;
       this.bytesReadCounter = bytesReadCounter;
       this.fileLength = length;


Mime
View raw message