tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3493. DAG submit timeout cannot be set to a month (Hitesh Shah via jeagles)
Date Fri, 04 Nov 2016 02:50:29 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 c0a5db22a -> 1b315ef9c


TEZ-3493. DAG submit timeout cannot be set to a month (Hitesh Shah via jeagles)

(cherry picked from commit ad68f73583681f79d825381d3bed8b80867d87e1)
(cherry picked from commit d9a698e4969b9eac9147ff9f2e337bd0ff9b1a73)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1b315ef9
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1b315ef9
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1b315ef9

Branch: refs/heads/branch-0.7
Commit: 1b315ef9c14733497d754dc272bdbff99c7d3a12
Parents: c0a5db2
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Thu Nov 3 20:50:10 2016 -0500
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Thu Nov 3 21:49:53 2016 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/common/TezCommonUtils.java   | 15 ++++++++++++
 .../apache/tez/dag/api/TezConfiguration.java    |  3 ++-
 .../apache/tez/common/TestTezCommonUtils.java   | 24 ++++++++++++++++++++
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  8 +++----
 5 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/1b315ef9/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8505f5b..e6f1a5e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3493. DAG submit timeout cannot be set to a month
   TEZ-3505. Move license to the file header for TezBytesWritableSerialization
   TEZ-3486. COMBINE_OUTPUT_RECORDS/COMBINE_INPUT_RECORDS are not correct
   TEZ-3487. Improvements in travis yml file to get builds to work.

http://git-wip-us.apache.org/repos/asf/tez/blob/1b315ef9/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
index e4cf028..707a918 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
@@ -459,4 +459,19 @@ public class TezCommonUtils {
     jobToken.write(jobToken_dob);
     return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength());
   }
+
+  public static long getDAGSessionTimeout(Configuration conf) {
+    int timeoutSecs = conf.getInt(
+        TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
+        TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT);
+    if (timeoutSecs < 0) {
+      return -1;
+    }
+    // Handle badly configured value to minimize impact of a spinning thread
+    if (timeoutSecs == 0) {
+      timeoutSecs = 1;
+    }
+    return 1000l * timeoutSecs;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/1b315ef9/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 9b32b5f..5b3075a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1044,7 +1044,8 @@ public class TezConfiguration extends Configuration {
 
   /**
    * Int value. Time (in seconds) for which the Tez AM should wait for a DAG to be submitted
before
-   * shutting down. Only relevant in session mode.
+   * shutting down. Only relevant in session mode. Any negative value will disable this check
and
+   * allow the AM to hang around forever in idle mode.
    */
   @ConfigurationScope(Scope.AM)
   public static final String TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS =

http://git-wip-us.apache.org/repos/asf/tez/blob/1b315ef9/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
index a7e6069..971c6ed 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
@@ -314,4 +314,28 @@ public class TestTezCommonUtils {
 
   }
 
+  @Test(timeout=5000)
+  public void testGetDAGSessionTimeout() {
+    Configuration conf = new Configuration(false);
+    Assert.assertEquals(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT*1000,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+    // set to 1 month - * 1000 guaranteed to cross positive integer boundary
+    conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
+        24 * 60 * 60 * 30);
+    Assert.assertEquals(86400l*1000*30,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+    // set to negative val
+    conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
+        -24 * 60 * 60 * 30);
+    Assert.assertEquals(-1,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+    conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 0);
+    Assert.assertEquals(1000,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/1b315ef9/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 98e9355..a84bc02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -539,9 +539,7 @@ public class DAGAppMaster extends AbstractService {
     historyEventHandler = createHistoryEventHandler(context);
     addIfService(historyEventHandler, true);
 
-    this.sessionTimeoutInterval = 1000 * amConf.getInt(
-            TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
-            TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT);
+    this.sessionTimeoutInterval = TezCommonUtils.getDAGSessionTimeout(amConf);
 
     recoveryDataDir = TezCommonUtils.getRecoveryPath(tezSystemStagingDir, conf);
     recoveryFS = recoveryDataDir.getFileSystem(conf);
@@ -1940,8 +1938,8 @@ public class DAGAppMaster extends AbstractService {
       }
     }
 
-    if (isSession) {
-      this.dagSubmissionTimer = new Timer(true);
+    if (isSession && sessionTimeoutInterval >= 0) {
+      this.dagSubmissionTimer = new Timer("DAGSubmissionTimer", true);
       this.dagSubmissionTimer.scheduleAtFixedRate(new TimerTask() {
         @Override
         public void run() {


Mime
View raw message