tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1355. Read host and shuffle meta-information for events only if data is generated by Outputs. Contributed by Jonathan Eagles.
Date Fri, 01 Aug 2014 21:25:32 GMT
Repository: tez
Updated Branches:
  refs/heads/master 0306928f7 -> c38d3127a


TEZ-1355. Read host and shuffle meta-information for events only if data is
generated by Outputs. Contributed by Jonathan Eagles.


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c38d3127
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c38d3127
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c38d3127

Branch: refs/heads/master
Commit: c38d3127ac3c2335fa7961e7a2395e4c5aba72d8
Parents: 0306928
Author: Siddharth Seth <sseth@apache.org>
Authored: Fri Aug 1 14:24:16 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Fri Aug 1 14:24:16 2014 -0700

----------------------------------------------------------------------
 .../tez/runtime/library/output/OnFileSortedOutput.java   | 11 +++++------
 .../runtime/library/output/OnFileUnorderedKVOutput.java  | 10 +++++-----
 2 files changed, 10 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/c38d3127/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
index 8536746..6aa319b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
@@ -149,12 +149,6 @@ public class OnFileSortedOutput extends AbstractLogicalOutput {
   }
   
   protected List<Event> generateEventsOnClose() throws IOException {
-    String host = System.getenv(ApplicationConstants.Environment.NM_HOST
-        .toString());
-    ByteBuffer shuffleMetadata = getContext()
-        .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
-    int shufflePort = ShuffleUtils.deserializeShuffleProviderMetaData(shuffleMetadata);
-
     DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto
         .newBuilder();
 
@@ -182,6 +176,11 @@ public class OnFileSortedOutput extends AbstractLogicalOutput {
       }
     }
     if (!sendEmptyPartitionDetails || outputGenerated) {
+      String host = System.getenv(ApplicationConstants.Environment.NM_HOST
+          .toString());
+      ByteBuffer shuffleMetadata = getContext()
+          .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
+      int shufflePort = ShuffleUtils.deserializeShuffleProviderMetaData(shuffleMetadata);
       payloadBuilder.setHost(host);
       payloadBuilder.setPort(shufflePort);
       payloadBuilder.setPathComponent(getContext().getUniqueIdentifier());

http://git-wip-us.apache.org/repos/asf/tez/blob/c38d3127/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
index 32592bb..2636c5c 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedKVOutput.java
@@ -127,11 +127,6 @@ public class OnFileUnorderedKVOutput extends AbstractLogicalOutput {
       payloadBuilder.setData(dataProtoBuilder.build());
     }
 
-    String host = getHost();
-    ByteBuffer shuffleMetadata = getContext()
-        .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
-    int shufflePort = ShuffleUtils
-        .deserializeShuffleProviderMetaData(shuffleMetadata);
     // Set the list of empty partitions - single partition on this case.
     if (!outputGenerated) {
       LOG.info("No output was generated");
@@ -142,6 +137,11 @@ public class OnFileUnorderedKVOutput extends AbstractLogicalOutput {
       payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
     }
     if (outputGenerated) {
+      String host = getHost();
+      ByteBuffer shuffleMetadata = getContext()
+          .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
+      int shufflePort = ShuffleUtils
+          .deserializeShuffleProviderMetaData(shuffleMetadata);
       payloadBuilder.setHost(host);
       payloadBuilder.setPort(shufflePort);
       payloadBuilder.setPathComponent(getContext().getUniqueIdentifier());


Mime
View raw message