tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3761. NPE in Fetcher under load (jeagles)
Date Fri, 23 Jun 2017 14:58:34 GMT
Repository: tez
Updated Branches:
  refs/heads/master 4a7719b0c -> a7f93ae1d


TEZ-3761. NPE in Fetcher under load (jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a7f93ae1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a7f93ae1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a7f93ae1

Branch: refs/heads/master
Commit: a7f93ae1da114fa03f7e500a12706e5737b189ee
Parents: 4a7719b
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Fri Jun 23 09:56:00 2017 -0500
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Fri Jun 23 09:56:00 2017 -0500

----------------------------------------------------------------------
 .../tez/runtime/library/common/shuffle/Fetcher.java       | 10 ++++++++++
 1 file changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/a7f93ae1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
index fefd53e..bf8c83b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
@@ -853,7 +853,17 @@ public class Fetcher extends CallableWithNdc<FetchResult> {
           ShuffleHeader header = new ShuffleHeader();
           header.readFields(input);
           pathComponent = header.getMapId();
+          if (!pathComponent.startsWith(InputAttemptIdentifier.PATH_PREFIX)) {
+            throw new IllegalArgumentException("Invalid map id: " + header.getMapId() + ",
expected to start with " +
+                InputAttemptIdentifier.PATH_PREFIX + ", partition: " + header.getPartition()
+                + " while fetching " + inputAttemptIdentifier);
+          }
+
           srcAttemptId = pathToAttemptMap.get(new PathPartition(pathComponent, header.getPartition()));
+          if (srcAttemptId == null) {
+            throw new IllegalArgumentException("Source attempt not found for map id: " +
header.getMapId() +
+                ", partition: " + header.getPartition() + " while fetching " + inputAttemptIdentifier);
+          }
 
           if (header.getCompressedLength() == 0) {
             // Empty partitions are already accounted for


Mime
View raw message