Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 27752180F6 for ; Fri, 26 Feb 2016 21:59:39 +0000 (UTC) Received: (qmail 48799 invoked by uid 500); 26 Feb 2016 21:59:39 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 48765 invoked by uid 500); 26 Feb 2016 21:59:39 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 48755 invoked by uid 99); 26 Feb 2016 21:59:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Feb 2016 21:59:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E81FDE0211; Fri, 26 Feb 2016 21:59:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jlowe@apache.org To: commits@tez.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout (jlowe) (cherry picked from commit 41ac55155a59ba60b3da64b6b627c5760d8d6dcf) Date: Fri, 26 Feb 2016 21:59:38 +0000 (UTC) Repository: tez Updated Branches: refs/heads/branch-0.7 20763f5b3 -> 1832554c7 TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout (jlowe) (cherry picked from commit 41ac55155a59ba60b3da64b6b627c5760d8d6dcf) Conflicts: CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1832554c Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1832554c Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1832554c Branch: refs/heads/branch-0.7 Commit: 1832554c77b2fa7f127e6d2bad183a23a96d3bf6 Parents: 20763f5 Author: Jason Lowe Authored: Fri Feb 26 21:59:16 2016 +0000 Committer: Jason Lowe Committed: Fri Feb 26 21:59:16 2016 +0000 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java | 4 ++-- .../org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java | 6 ------ 3 files changed, 3 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/1832554c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 99f1b16..169b1f4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES TEZ-2972. Avoid task rescheduling when a node turns unhealthy ALL CHANGES: + TEZ-3141. mapreduce.task.timeout is not translated to container heartbeat timeout TEZ-3128. Avoid stopping containers on the AM shutdown thread. TEZ-3129. Tez task and task attempt UI needs application fails with NotFoundException TEZ-3114. Shuffle OOM due to EventMetaData flood http://git-wip-us.apache.org/repos/asf/tez/blob/1832554c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 345f7aa..1335aa2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -82,6 +82,8 @@ public class DeprecatedKeys { TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH); mrParamToDAGParamMap.put(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH); + mrParamToDAGParamMap.put(MRJobConfig.TASK_TIMEOUT, + TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); } // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. @@ -153,8 +155,6 @@ public class DeprecatedKeys { registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS); registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC); - - registerMRToRuntimeKeyTranslation(MRJobConfig.TASK_TIMEOUT, TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); } private static void addDeprecatedKeys() { http://git-wip-us.apache.org/repos/asf/tez/blob/1832554c/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java index 06d2990..ae218a0 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java @@ -43,7 +43,6 @@ public class TestDeprecatedKeys { jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f); jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true); jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f); - jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000); MRHelpers.translateMRConfToTez(jobConf); @@ -65,7 +64,6 @@ public class TestDeprecatedKeys { assertEquals(0.33f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0), 0.01f); - assertEquals(1000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000)); } @Test(timeout = 5000) @@ -78,7 +76,6 @@ public class TestDeprecatedKeys { jobConf.setInt(MRJobConfig.IO_SORT_MB, 100); jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100); jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f); - jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000); jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000); jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200); @@ -103,7 +100,6 @@ public class TestDeprecatedKeys { jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator"); jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator"); - jobConf.setInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000); jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false); jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true); @@ -135,7 +131,6 @@ public class TestDeprecatedKeys { assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "")); assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false)); assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0.0f), 0.0f); - assertEquals(2000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD)); assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES)); @@ -161,7 +156,6 @@ public class TestDeprecatedKeys { assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS)); assertNull(jobConf.get("map.sort.class")); assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART)); - assertNull(jobConf.get(MRJobConfig.TASK_TIMEOUT)); } }