tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1208. Log time taken to connect/getInputStream to a http source in fetcher. Contributed by Rajesh Balamohan.
Date Tue, 17 Jun 2014 19:51:58 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master 608ee831b -> ca4ec1896


TEZ-1208. Log time taken to connect/getInputStream to a http source in
fetcher. Contributed by Rajesh Balamohan.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/ca4ec189
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/ca4ec189
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/ca4ec189

Branch: refs/heads/master
Commit: ca4ec1896be2121d1694481b18b9c7fedef31eb6
Parents: 608ee83
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Jun 17 12:51:36 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Jun 17 12:51:36 2014 -0700

----------------------------------------------------------------------
 .../library/shuffle/common/HttpConnection.java  | 26 +++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ca4ec189/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
index 5b56d5b..1dd36b4 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/HttpConnection.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.GeneralSecurityException;
+import java.util.concurrent.TimeUnit;
 
 import javax.crypto.SecretKey;
 import javax.net.ssl.HttpsURLConnection;
@@ -38,6 +39,8 @@ import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.tez.runtime.library.common.security.SecureShuffleUtils;
 import org.apache.tez.runtime.library.common.shuffle.impl.ShuffleHeader;
 
+import com.google.common.base.Stopwatch;
+
 /**
  * HttpConnection which can be used for Unordered / Ordered shuffle.
  */
@@ -65,6 +68,7 @@ public class HttpConnection {
   private String msgToEncode;
 
   private final HttpConnectionParams httpConnParams;
+  private final Stopwatch stopWatch;
 
   /**
    * HttpConnection
@@ -81,6 +85,7 @@ public class HttpConnection {
     this.jobTokenSecret = jobTokenSecret;
     this.httpConnParams = connParams;
     this.url = url;
+    this.stopWatch = new Stopwatch();
     if (LOG.isDebugEnabled()) {
       LOG.debug("MapOutput URL :" + url.toString());
     }
@@ -132,6 +137,7 @@ public class HttpConnection {
    * @throws IOException
    */
   public boolean connect(int connectionTimeout) throws IOException {
+    stopWatch.reset().start();
     if (connection == null) {
       setupConnection();
     }
@@ -144,6 +150,7 @@ public class HttpConnection {
     }
     // set the connect timeout to the unit-connect-timeout
     connection.setConnectTimeout(unit);
+    int connectionFailures = 0;
     while (true) {
       try {
         connection.connect();
@@ -170,12 +177,18 @@ public class HttpConnection {
           // reset the connect time out for the final connect
           connection.setConnectTimeout(unit);
         }
+        connectionFailures++;
       }
     }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Time taken to connect to " + url.toString() +
+        " " + stopWatch.elapsedTime(TimeUnit.MILLISECONDS) + " ms; connectionFailures="+
connectionFailures);
+    }
     return true;
   }
 
   public void validate() throws IOException {
+    stopWatch.reset().start();
     int rc = connection.getResponseCode();
     if (rc != HttpURLConnection.HTTP_OK) {
       throw new IOException("Got invalid response code " + rc + " from " + url
@@ -199,7 +212,8 @@ public class HttpConnection {
         + replyHash);
     // verify that replyHash is HMac of encHash
     SecureShuffleUtils.verifyReply(replyHash, encHash, jobTokenSecret);
-    LOG.info("for url=" + msgToEncode + " sent hash and receievd reply");
+    LOG.info("for url=" + msgToEncode +
+      " sent hash and receievd reply " + stopWatch.elapsedTime(TimeUnit.MILLISECONDS) + "
ms");
   }
 
   /**
@@ -209,12 +223,17 @@ public class HttpConnection {
    * @throws IOException
    */
   public DataInputStream getInputStream() throws IOException {
+    stopWatch.reset().start();
     DataInputStream input = null;
     if (connectionSucceeed) {
       input =
           new DataInputStream(new BufferedInputStream(
             connection.getInputStream(), httpConnParams.bufferSize));
     }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Time taken to getInputStream (connect) " + url +
+        " " + stopWatch.elapsedTime(TimeUnit.MILLISECONDS) + " ms");
+    }
     return input;
   }
 
@@ -237,6 +256,7 @@ public class HttpConnection {
    */
   public void cleanup(boolean disconnect) throws IOException {
     cleanup = true;
+    stopWatch.reset().start();
     try {
       if (input != null) {
         LOG.info("Closing input on " + logIdentifier);
@@ -259,6 +279,10 @@ public class HttpConnection {
             + ": " + e.getMessage());
       }
     }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Time taken to cleanup connection to " + url +
+        " " + stopWatch.elapsedTime(TimeUnit.MILLISECONDS) + " ms");
+    }
   }
 
   /**


Mime
View raw message