tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-634. Reduce frequency of getTask logging. (sseth)
Date Mon, 25 Nov 2013 20:58:08 GMT
Updated Branches:
  refs/heads/master d2d397aaa -> 3ec17f6d8


TEZ-634. Reduce frequency of getTask logging. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/3ec17f6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/3ec17f6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/3ec17f6d

Branch: refs/heads/master
Commit: 3ec17f6d84584ce2d7b6e7c3347dac0aaaf9da77
Parents: d2d397a
Author: Siddharth Seth <sseth@apache.org>
Authored: Mon Nov 25 12:57:49 2013 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Mon Nov 25 12:57:49 2013 -0800

----------------------------------------------------------------------
 .../apache/hadoop/mapred/YarnTezDagChild.java   | 39 +++++++++++++-------
 1 file changed, 26 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/3ec17f6d/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
index 1adb5a8..6e77d37 100644
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
+++ b/tez-dag/src/main/java/org/apache/hadoop/mapred/YarnTezDagChild.java
@@ -362,9 +362,9 @@ public class YarnTezDagChild {
 
     // report non-pid to application master
     String pid = System.getenv().get("JVM_PID");
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("PID, containerId: " + pid + ", " + containerIdentifier);
-    }
+    
+    LOG.info("PID, containerIdentifier: " + pid + ", " + containerIdentifier);
+    
     ContainerTask containerTask = null;
     UserGroupInformation childUGI = null;
     ContainerContext containerContext = new ContainerContext(
@@ -381,19 +381,32 @@ public class YarnTezDagChild {
         if (taskCount > 0) {
           updateLoggers(null);
         }
+        boolean isNewGetTask = true;
+        long getTaskPollStartTime = System.currentTimeMillis();
+        long nextGetTaskPrintTime = getTaskPollStartTime + 2000l;
         for (int idle = 0; null == containerTask; ++idle) {
-          long sleepTimeMilliSecs = Math.min(idle * 10, getTaskMaxSleepTime);
-          LOG.info("Sleeping for " + sleepTimeMilliSecs
-              + "ms before retrying getTask again. Got null now.");
-          MILLISECONDS.sleep(sleepTimeMilliSecs);
+          if (!isNewGetTask) { // Don't sleep on the first iteration.
+            long sleepTimeMilliSecs = Math.min(idle * 10, getTaskMaxSleepTime);
+            if (sleepTimeMilliSecs + System.currentTimeMillis() > nextGetTaskPrintTime)
{
+              LOG.info("Sleeping for "
+                  + sleepTimeMilliSecs
+                  + "ms before retrying getTask again. Got null now. "
+                  + "Next getTask sleep message after 2s");
+              nextGetTaskPrintTime = System.currentTimeMillis() + sleepTimeMilliSecs + 2000l;
+            }
+            MILLISECONDS.sleep(sleepTimeMilliSecs);
+          } else {
+            LOG.info("Attempting to fetch new task");
+          }
+          isNewGetTask = false;
           containerTask = umbilical.getTask(containerContext);
         }
-        LOG.info("TaskInfo: shouldDie: "
-            + containerTask.shouldDie()
-            + (containerTask.shouldDie() == true ?
-                "" : ", currentTaskAttemptId: "
-                  + containerTask.getTaskSpec().getTaskAttemptID()));
-
+        LOG.info("Got TaskUpdate: "
+            + (System.currentTimeMillis() - getTaskPollStartTime)
+            + " ms after starting to poll."
+            + " TaskInfo: shouldDie: " + containerTask.shouldDie()
+            + (containerTask.shouldDie() == true ? "" : ", currentTaskAttemptId: "
+                + containerTask.getTaskSpec().getTaskAttemptID()));
         if (containerTask.shouldDie()) {
           return;
         }


Mime
View raw message