tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject git commit: TEZ-1609. Add hostname to logIdentifiers of fetchers for easy debugging (Rajesh Balamohan)
Date Fri, 26 Sep 2014 22:57:43 GMT
Repository: tez
Updated Branches:
  refs/heads/master 160a5d80f -> fa2928f6a


TEZ-1609. Add hostname to logIdentifiers of fetchers for easy debugging  (Rajesh Balamohan)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fa2928f6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fa2928f6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fa2928f6

Branch: refs/heads/master
Commit: fa2928f6a9fae32fb7edc5dad6374ca625c3b417
Parents: 160a5d8
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Sat Sep 27 04:27:31 2014 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Sat Sep 27 04:27:31 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                                  | 1 +
 .../tez/runtime/library/common/shuffle/impl/Fetcher.java     | 5 ++++-
 .../tez/runtime/library/shuffle/common/HttpConnection.java   | 8 +++++---
 .../runtime/library/shuffle/common/impl/ShuffleManager.java  | 4 +++-
 4 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e96d7c6..741c99a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -70,6 +70,7 @@ ALL CHANGES
   TEZ-1433. Invalid credentials can be used when a DAG is submitted to a
   session which has timed out
   TEZ-1624. Flaky tests in TestContainerReuse due to race condition in DelayedContainerManager
thread
+  TEZ-1609. Add hostname to logIdentifiers of fetchers for easy debugging
 
 Release 0.5.0: 2014-09-03
 

http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
index 9b009ea..c29401b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/Fetcher.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.tez.common.TezUtilsInternal;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.tez.common.TezRuntimeFrameworkConfigs;
@@ -92,6 +93,8 @@ class Fetcher extends Thread {
 
   HttpConnection httpConnection;
   HttpConnectionParams httpConnectionParams;
+
+  final static String localhostName = NetUtils.getHostname();
   
   public Fetcher(HttpConnectionParams httpConnectionParams,
                  ShuffleScheduler scheduler, MergeManager merger,
@@ -132,7 +135,7 @@ class Fetcher extends Thread {
     this.localDiskFetchEnabled = localDiskFetchEnabled;
 
     this.logIdentifier = "fetcher [" + TezUtilsInternal
-        .cleanVertexName(inputContext.getSourceVertexName()) + "] #" + id;
+        .cleanVertexName(inputContext.getSourceVertexName()) + "] #" + id + " " + localhostName;
     setName(logIdentifier);
     setDaemon(true);
   }  

http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
index d664f88..1311a4f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
@@ -210,11 +210,13 @@ public class HttpConnection {
     if (replyHash == null) {
       throw new IOException("security validation of TT Map output failed");
     }
-    LOG.debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash="
-        + replyHash);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash="
+          + replyHash);
+    }
     // verify that replyHash is HMac of encHash
     SecureShuffleUtils.verifyReply(replyHash, encHash, jobTokenSecret);
-    LOG.info("for url=" + msgToEncode +
+    LOG.info("for url=" + url +
       " sent hash and receievd reply " + stopWatch.elapsedTime(TimeUnit.MILLISECONDS) + "
ms");
   }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/fa2928f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
index 8aa6582..ddf98b6 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.tez.common.TezRuntimeFrameworkConfigs;
 import org.apache.tez.common.TezUtilsInternal;
 import org.apache.tez.common.counters.TaskCounter;
@@ -153,6 +154,7 @@ public class ShuffleManager implements FetcherCallback {
   private final LocalDirAllocator localDirAllocator;
   private final RawLocalFileSystem localFs;
   private final Path[] localDisks;
+  private final static String localhostName = NetUtils.getHostname();
 
   // TODO More counters - FetchErrors, speed?
   
@@ -199,7 +201,7 @@ public class ShuffleManager implements FetcherCallback {
     ExecutorService fetcherRawExecutor = Executors.newFixedThreadPool(
         numFetchers,
         new ThreadFactoryBuilder().setDaemon(true)
-            .setNameFormat("Fetcher [" + srcNameTrimmed + "] #%d").build());
+            .setNameFormat("Fetcher [" + srcNameTrimmed + "] #%d " + localhostName).build());
     this.fetcherExecutor = MoreExecutors.listeningDecorator(fetcherRawExecutor);
     
     ExecutorService schedulerRawExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder()


Mime
View raw message