tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject [28/50] [abbrv] tez git commit: TEZ-3606. Fix debug log for empty partitions to the expanded partitionId in the Composite case (Kuhu Shukla via jeagles)
Date Wed, 24 May 2017 21:07:57 GMT
TEZ-3606. Fix debug log for empty partitions to the expanded partitionId in the Composite case
(Kuhu Shukla via jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/de2e0f26
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/de2e0f26
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/de2e0f26

Branch: refs/heads/master
Commit: de2e0f268750c147e583eb62d1e8ae5606590cd2
Parents: 29d6df2
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Fri Feb 3 15:05:14 2017 -0600
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Fri Feb 3 15:05:14 2017 -0600

----------------------------------------------------------------------
 TEZ-3334-CHANGES.txt                                          | 1 +
 .../common/shuffle/impl/ShuffleInputEventHandlerImpl.java     | 7 ++++---
 .../ShuffleInputEventHandlerOrderedGrouped.java               | 7 ++++---
 3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/de2e0f26/TEZ-3334-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt
index 49da5aa..1ba8ca6 100644
--- a/TEZ-3334-CHANGES.txt
+++ b/TEZ-3334-CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
 INCOMPATIBLE CHANGES:
 
 ALL CHANGES:
+  TEZ-3606. Fix debug log for empty partitions to the expanded partitionId in the Composite
case
   TEZ-3604. Remove the compositeInputAttemptIdentifier from remaining list upon fetch completion
in the Ordered case
   TEZ-3599. Unordered Fetcher can hang if empty partitions are present
   TEZ-3596. Number of Empty DME logged for Composite fetch is too high

http://git-wip-us.apache.org/repos/asf/tez/blob/de2e0f26/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
index ee06de5..542ec34 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
@@ -201,7 +201,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler
{
     }
 
     if (shufflePayload.hasEmptyPartitions()) {
-      CompositeInputAttemptIdentifier srcAttemptIdentifier =
+      CompositeInputAttemptIdentifier compositeInputAttemptIdentifier =
           constructInputAttemptIdentifier(crdme.getTargetIndex(), crdme.getCount(), crdme.getVersion(),
shufflePayload, false);
 
       boolean allPartitionsEmpty = true;
@@ -209,12 +209,13 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler
{
         int srcPartitionId = partitionId + i;
         allPartitionsEmpty &= emptyPartitionsBitSet.get(srcPartitionId);
         if (emptyPartitionsBitSet.get(srcPartitionId)) {
+          InputAttemptIdentifier srcAttemptIdentifier = compositeInputAttemptIdentifier.expand(i);
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Source partition: " + partitionId + " did not generate any data. SrcAttempt:
["
+            LOG.debug("Source partition: " + srcPartitionId + " did not generate any data.
SrcAttempt: ["
                 + srcAttemptIdentifier + "]. Not fetching.");
           }
           numDmeEventsNoData.getAndIncrement();
-          shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier.expand(i));
+          shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier);
         }
       }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/de2e0f26/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
index cc24db9..fda899f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
@@ -190,12 +190,13 @@ public class ShuffleInputEventHandlerOrderedGrouped implements ShuffleEventHandl
         int srcPartitionId = partitionId + i;
         allPartitionsEmpty &= emptyPartitionsBitSet.get(srcPartitionId);
         if (emptyPartitionsBitSet.get(srcPartitionId)) {
+          InputAttemptIdentifier srcInputAttemptIdentifier = compositeInputAttemptIdentifier.expand(i);
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Source partition: " + partitionId + " did not generate any data. SrcAttempt:
["
-                + compositeInputAttemptIdentifier + "]. Not fetching.");
+            LOG.debug("Source partition: " + srcPartitionId + " did not generate any data.
SrcAttempt: ["
+                + srcInputAttemptIdentifier + "]. Not fetching.");
           }
           numDmeEventsNoData.getAndIncrement();
-          scheduler.copySucceeded(compositeInputAttemptIdentifier.expand(i), null, 0, 0,
0, null, true);
+          scheduler.copySucceeded(srcInputAttemptIdentifier, null, 0, 0, 0, null, true);
         }
       }
 


Mime
View raw message