Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 130A9200BC3 for ; Fri, 4 Nov 2016 02:50:52 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 1010E160B0B; Fri, 4 Nov 2016 01:50:52 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 32BC0160AFF for ; Fri, 4 Nov 2016 02:50:51 +0100 (CET) Received: (qmail 9994 invoked by uid 500); 4 Nov 2016 01:50:50 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 9985 invoked by uid 99); 4 Nov 2016 01:50:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Nov 2016 01:50:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 51042E09DE; Fri, 4 Nov 2016 01:50:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jeagles@apache.org To: commits@tez.apache.org Message-Id: <3708b42ef4344b7286739841b5d66109@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-3493. DAG submit timeout cannot be set to a month (Hitesh Shah via jeagles) Date: Fri, 4 Nov 2016 01:50:50 +0000 (UTC) archived-at: Fri, 04 Nov 2016 01:50:52 -0000 Repository: tez Updated Branches: refs/heads/master bebbfa4aa -> ad68f7358 TEZ-3493. DAG submit timeout cannot be set to a month (Hitesh Shah via jeagles) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ad68f735 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ad68f735 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ad68f735 Branch: refs/heads/master Commit: ad68f73583681f79d825381d3bed8b80867d87e1 Parents: bebbfa4 Author: Jonathan Eagles Authored: Thu Nov 3 20:50:10 2016 -0500 Committer: Jonathan Eagles Committed: Thu Nov 3 20:50:10 2016 -0500 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../org/apache/tez/common/TezCommonUtils.java | 14 ++++++++++++ .../apache/tez/dag/api/TezConfiguration.java | 3 ++- .../apache/tez/common/TestTezCommonUtils.java | 24 ++++++++++++++++++++ .../org/apache/tez/dag/app/DAGAppMaster.java | 6 ++--- 5 files changed, 45 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 2ee8026..033291a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3493. DAG submit timeout cannot be set to a month TEZ-3505. Move license to the file header for TezBytesWritableSerialization TEZ-3486. COMBINE_OUTPUT_RECORDS/COMBINE_INPUT_RECORDS are not correct TEZ-3247. Add more unit test coverage for container reuse. @@ -141,6 +142,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3493. DAG submit timeout cannot be set to a month TEZ-3505. Move license to the file header for TezBytesWritableSerialization TEZ-3486. COMBINE_OUTPUT_RECORDS/COMBINE_INPUT_RECORDS are not correct TEZ-3097. Flaky test: TestCommit.testDAGCommitStartedEventFail_OnDAGSuccess. @@ -644,6 +646,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3493. DAG submit timeout cannot be set to a month TEZ-3505. Move license to the file header for TezBytesWritableSerialization TEZ-3486. COMBINE_OUTPUT_RECORDS/COMBINE_INPUT_RECORDS are not correct TEZ-3437. Improve synchronization and the progress report behavior for Inputs from TEZ-3317. http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java index 69e48b2..9cb76d9 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java @@ -545,4 +545,18 @@ public class TezCommonUtils { heartbeatIntervalMillis/buckets); } + public static long getDAGSessionTimeout(Configuration conf) { + int timeoutSecs = conf.getInt( + TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, + TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT); + if (timeoutSecs < 0) { + return -1; + } + // Handle badly configured value to minimize impact of a spinning thread + if (timeoutSecs == 0) { + timeoutSecs = 1; + } + return 1000l * timeoutSecs; + } + } http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index c4272b7..a09e888 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1139,7 +1139,8 @@ public class TezConfiguration extends Configuration { /** * Int value. Time (in seconds) for which the Tez AM should wait for a DAG to be submitted before - * shutting down. Only relevant in session mode. + * shutting down. Only relevant in session mode. Any negative value will disable this check and + * allow the AM to hang around forever in idle mode. */ @ConfigurationScope(Scope.AM) @ConfigurationProperty(type="integer") http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java index 5f0b33b..3929c4b 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java @@ -384,5 +384,29 @@ public class TestTezCommonUtils { Assert.assertFalse(value.contains(version)); } + @Test(timeout=5000) + public void testGetDAGSessionTimeout() { + Configuration conf = new Configuration(false); + Assert.assertEquals(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT*1000, + TezCommonUtils.getDAGSessionTimeout(conf)); + + // set to 1 month - * 1000 guaranteed to cross positive integer boundary + conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, + 24 * 60 * 60 * 30); + Assert.assertEquals(86400l*1000*30, + TezCommonUtils.getDAGSessionTimeout(conf)); + + // set to negative val + conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, + -24 * 60 * 60 * 30); + Assert.assertEquals(-1, + TezCommonUtils.getDAGSessionTimeout(conf)); + + conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 0); + Assert.assertEquals(1000, + TezCommonUtils.getDAGSessionTimeout(conf)); + + } + } http://git-wip-us.apache.org/repos/asf/tez/blob/ad68f735/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 062f29d..7bf5d26 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -604,9 +604,7 @@ public class DAGAppMaster extends AbstractService { historyEventHandler = createHistoryEventHandler(context); addIfService(historyEventHandler, true); - this.sessionTimeoutInterval = 1000 * amConf.getInt( - TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, - TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT); + this.sessionTimeoutInterval = TezCommonUtils.getDAGSessionTimeout(amConf); this.clientAMHeartbeatTimeoutIntervalMillis = TezCommonUtils.getAMClientHeartBeatTimeoutMillis(amConf); @@ -2118,7 +2116,7 @@ public class DAGAppMaster extends AbstractService { } } - if (isSession) { + if (isSession && sessionTimeoutInterval >= 0) { this.dagSubmissionTimer = new Timer("DAGSubmissionTimer", true); this.dagSubmissionTimer.scheduleAtFixedRate(new TimerTask() { @Override