tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject [33/50] [abbrv] tez git commit: TEZ-3493. DAG submit timeout cannot be set to a month (Hitesh Shah via jeagles)
Date Tue, 06 Dec 2016 17:07:04 GMT
TEZ-3493. DAG submit timeout cannot be set to a month (Hitesh Shah via jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ad68f735
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ad68f735
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ad68f735

Branch: refs/heads/TEZ-3334
Commit: ad68f73583681f79d825381d3bed8b80867d87e1
Parents: bebbfa4
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Thu Nov 3 20:50:10 2016 -0500
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Thu Nov 3 20:50:10 2016 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +++
 .../org/apache/tez/common/TezCommonUtils.java   | 14 ++++++++++++
 .../apache/tez/dag/api/TezConfiguration.java    |  3 ++-
 .../apache/tez/common/TestTezCommonUtils.java   | 24 ++++++++++++++++++++
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  6 ++---
 5 files changed, 45 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2ee8026..033291a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3493. DAG submit timeout cannot be set to a month
   TEZ-3505. Move license to the file header for TezBytesWritableSerialization
   TEZ-3486. COMBINE_OUTPUT_RECORDS/COMBINE_INPUT_RECORDS are not correct
   TEZ-3247. Add more unit test coverage for container reuse.
@@ -141,6 +142,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3493. DAG submit timeout cannot be set to a month
   TEZ-3505. Move license to the file header for TezBytesWritableSerialization
   TEZ-3486. COMBINE_OUTPUT_RECORDS/COMBINE_INPUT_RECORDS are not correct
   TEZ-3097. Flaky test: TestCommit.testDAGCommitStartedEventFail_OnDAGSuccess.
@@ -644,6 +646,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3493. DAG submit timeout cannot be set to a month
   TEZ-3505. Move license to the file header for TezBytesWritableSerialization
   TEZ-3486. COMBINE_OUTPUT_RECORDS/COMBINE_INPUT_RECORDS are not correct
   TEZ-3437. Improve synchronization and the progress report behavior for Inputs from TEZ-3317.

http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
index 69e48b2..9cb76d9 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
@@ -545,4 +545,18 @@ public class TezCommonUtils {
         heartbeatIntervalMillis/buckets);
   }
 
+  public static long getDAGSessionTimeout(Configuration conf) {
+    int timeoutSecs = conf.getInt(
+        TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
+        TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT);
+    if (timeoutSecs < 0) {
+      return -1;
+    }
+    // Handle badly configured value to minimize impact of a spinning thread
+    if (timeoutSecs == 0) {
+      timeoutSecs = 1;
+    }
+    return 1000l * timeoutSecs;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index c4272b7..a09e888 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1139,7 +1139,8 @@ public class TezConfiguration extends Configuration {
 
   /**
    * Int value. Time (in seconds) for which the Tez AM should wait for a DAG to be submitted
before
-   * shutting down. Only relevant in session mode.
+   * shutting down. Only relevant in session mode. Any negative value will disable this check
and
+   * allow the AM to hang around forever in idle mode.
    */
   @ConfigurationScope(Scope.AM)
   @ConfigurationProperty(type="integer")

http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
index 5f0b33b..3929c4b 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
@@ -384,5 +384,29 @@ public class TestTezCommonUtils {
     Assert.assertFalse(value.contains(version));
   }
 
+  @Test(timeout=5000)
+  public void testGetDAGSessionTimeout() {
+    Configuration conf = new Configuration(false);
+    Assert.assertEquals(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT*1000,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+    // set to 1 month - * 1000 guaranteed to cross positive integer boundary
+    conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
+        24 * 60 * 60 * 30);
+    Assert.assertEquals(86400l*1000*30,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+    // set to negative val
+    conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
+        -24 * 60 * 60 * 30);
+    Assert.assertEquals(-1,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+    conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 0);
+    Assert.assertEquals(1000,
+        TezCommonUtils.getDAGSessionTimeout(conf));
+
+  }
+
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 062f29d..7bf5d26 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -604,9 +604,7 @@ public class DAGAppMaster extends AbstractService {
     historyEventHandler = createHistoryEventHandler(context);
     addIfService(historyEventHandler, true);
 
-    this.sessionTimeoutInterval = 1000 * amConf.getInt(
-            TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS,
-            TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT);
+    this.sessionTimeoutInterval = TezCommonUtils.getDAGSessionTimeout(amConf);
     this.clientAMHeartbeatTimeoutIntervalMillis =
         TezCommonUtils.getAMClientHeartBeatTimeoutMillis(amConf);
 
@@ -2118,7 +2116,7 @@ public class DAGAppMaster extends AbstractService {
       }
     }
 
-    if (isSession) {
+    if (isSession && sessionTimeoutInterval >= 0) {
       this.dagSubmissionTimer = new Timer("DAGSubmissionTimer", true);
       this.dagSubmissionTimer.scheduleAtFixedRate(new TimerTask() {
         @Override


Mime
View raw message