tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1048. Fix an NPE which can occur when the source task generates no data for a partition, and runs multiple attempts. (sseth)
Date Fri, 11 Apr 2014 22:20:43 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master a786e41a3 -> 150ebd3c7


TEZ-1048. Fix an NPE which can occur when the source task generates no
data for a partition, and runs multiple attempts. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/150ebd3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/150ebd3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/150ebd3c

Branch: refs/heads/master
Commit: 150ebd3c7934339f31a7b24012ece8e171e1f993
Parents: a786e41
Author: Siddharth Seth <sseth@apache.org>
Authored: Fri Apr 11 15:02:21 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Fri Apr 11 15:02:21 2014 -0700

----------------------------------------------------------------------
 .../library/common/shuffle/impl/ShuffleInputEventHandler.java  | 3 ++-
 .../runtime/library/common/shuffle/impl/ShuffleScheduler.java  | 6 +++++-
 .../shuffle/common/impl/ShuffleInputEventHandlerImpl.java      | 3 ++-
 3 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/150ebd3c/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
index cfad1fc..e0a4160 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandler.java
@@ -90,7 +90,8 @@ public class ShuffleInputEventHandler {
         byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions());
         BitSet emptyPartitionsBitSet = TezUtils.fromByteArray(emptyPartitions);
         if (emptyPartitionsBitSet.get(partitionId)) {
-          LOG.info("Source partition: " + partitionId + " did not generate any data. Not
fetching.");
+          LOG.info("Source partition: " + partitionId + " did not generate any data. SrcAttempt:
["
+              + srcAttemptIdentifier + "]. Not fetching.");
           scheduler.copySucceeded(srcAttemptIdentifier, null, 0, 0, 0, null);
           return;
         }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/150ebd3c/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
index c2bc903..45a50ad 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
@@ -203,7 +203,11 @@ class ShuffleScheduler {
       // input is already finished. duplicate fetch.
       LOG.warn("Duplicate fetch of input no longer needs to be fetched: " + srcAttemptIdentifier);
       // free the resource - specially memory
-      output.abort();
+      
+      // If the src does not generate data, output will be null.
+      if (output != null) {
+        output.abort();
+      }
     }
     // TODO NEWTEZ Should this be releasing the output, if not committed ? Possible memory
leak in case of speculation.
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/150ebd3c/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
index 09cd1ea..ac5da74 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleInputEventHandlerImpl.java
@@ -106,7 +106,8 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler
{
       if (emptyPartionsBitSet.get(srcIndex)) {
         InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(dme.getTargetIndex(),
             dme.getVersion());
-        LOG.info("Source partition: " + srcIndex + " did not generate any data. Not fetching.");
+        LOG.info("Source partition: " + srcIndex + " did not generate any data. SrcAttempt:
["
+            + srcAttemptIdentifier + "]. Not fetching.");
         shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier);
         return;
       }


Mime
View raw message