tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject tez git commit: TEZ-2907. NPE in IFile.Reader.getLength during final merge operation (rbalamohan)
Date Thu, 22 Oct 2015 21:13:56 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 35bd5ab0b -> 0bb3b1d23


TEZ-2907. NPE in IFile.Reader.getLength during final merge operation (rbalamohan)

(cherry picked from commit f3cea630e8329018a1e600859353576cb83cf77d)

# Conflicts:
#	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0bb3b1d2
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0bb3b1d2
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0bb3b1d2

Branch: refs/heads/branch-0.7
Commit: 0bb3b1d2300f02b62a0858c09f2835ca9d0bbdbe
Parents: 35bd5ab
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Thu Oct 22 13:59:05 2015 -0700
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Thu Oct 22 14:06:57 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                                    | 2 ++
 .../library/common/shuffle/orderedgrouped/MergeManager.java    | 6 ++++++
 2 files changed, 8 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/0bb3b1d2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 29e61bf..ba5b75d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.7.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES
+  TEZ-2907. NPE in IFile.Reader.getLength during final merge operation
   TEZ-2850. Tez MergeManager OOM for small Map Outputs
   TEZ-2886. Ability to merge AM credentials with DAG credentials.
   TEZ-2896. Fix thread names used during Input/Output initialization.
@@ -295,6 +296,7 @@ Release 0.6.3: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2907. NPE in IFile.Reader.getLength during final merge operation
   TEZ-2850. Tez MergeManager OOM for small Map Outputs
   TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails
   TEZ-2855. Fix a potential NPE while routing VertexManager events.

http://git-wip-us.apache.org/repos/asf/tez/blob/0bb3b1d2/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index e25c064..9516f27 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -846,12 +846,14 @@ public class MergeManager {
   class RawKVIteratorReader extends IFile.Reader {
 
     private final TezRawKeyValueIterator kvIter;
+    private final long size;
 
     public RawKVIteratorReader(TezRawKeyValueIterator kvIter, long size)
         throws IOException {
       super(null, size, null, spilledRecordsCounter, null, ifileReadAhead,
           ifileReadAheadLength, ifileBufferSize);
       this.kvIter = kvIter;
+      this.size = size;
     }
     @Override
     public KeyState readRawKey(DataInputBuffer key) throws IOException {
@@ -879,6 +881,10 @@ public class MergeManager {
     public void close() throws IOException {
       kvIter.close();
     }
+
+    @Override public long getLength() {
+      return size;
+    }
   }
 
   private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs,


Mime
View raw message