Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 9AC1A200BAB for ; Sat, 8 Oct 2016 08:10:13 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 996C9160AF4; Sat, 8 Oct 2016 06:10:13 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id BBB31160AE9 for ; Sat, 8 Oct 2016 08:10:12 +0200 (CEST) Received: (qmail 68663 invoked by uid 500); 8 Oct 2016 06:10:02 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 62112 invoked by uid 99); 8 Oct 2016 06:09:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 08 Oct 2016 06:09:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 553D3E78A1; Sat, 8 Oct 2016 06:09:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: drankye@apache.org To: common-commits@hadoop.apache.org Date: Sat, 08 Oct 2016 06:10:43 -0000 Message-Id: <10291e2c4ffc424fb917c60864bb27f0@git.apache.org> In-Reply-To: <021c9ac983d14fda98b768c867269e41@git.apache.org> References: <021c9ac983d14fda98b768c867269e41@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [47/50] [abbrv] hadoop git commit: MAPREDUCE-6776. yarn.app.mapreduce.client.job.max-retries should have a more useful default (miklos.szegedi@cloudera.com via rkanter) archived-at: Sat, 08 Oct 2016 06:10:13 -0000 MAPREDUCE-6776. yarn.app.mapreduce.client.job.max-retries should have a more useful default (miklos.szegedi@cloudera.com via rkanter) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3f37e6f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3f37e6f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3f37e6f Branch: refs/heads/HADOOP-12756 Commit: f3f37e6fb8172f6434e06eb9a137c0c155b3952e Parents: 2e853be Author: Robert Kanter Authored: Fri Oct 7 14:47:06 2016 -0700 Committer: Robert Kanter Committed: Fri Oct 7 14:47:06 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/mapreduce/MRJobConfig.java | 2 +- .../src/main/resources/mapred-default.xml | 10 +++--- .../apache/hadoop/mapred/JobClientUnitTest.java | 34 ++++++++++++++++---- 3 files changed, 34 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f37e6f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 5716404..1325b74 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -505,7 +505,7 @@ public interface MRJobConfig { */ public static final String MR_CLIENT_JOB_MAX_RETRIES = MR_PREFIX + "client.job.max-retries"; - public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 0; + public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 3; /** * How long to wait between jobclient retries on failure http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f37e6f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 73aaa7a..fe29212 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1505,12 +1505,12 @@ yarn.app.mapreduce.client.job.max-retries - 0 + 3 The number of retries the client will make for getJob and - dependent calls. The default is 0 as this is generally only needed for - non-HDFS DFS where additional, high level retries are required to avoid - spurious failures during the getJob call. 30 is a good value for - WASB + dependent calls. + This is needed for non-HDFS DFS where additional, high level + retries are required to avoid spurious failures during the getJob call. + 30 is a good value for WASB http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f37e6f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java index 4895a5b..e02232d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java @@ -225,10 +225,10 @@ public class JobClientUnitTest { //To prevent the test from running for a very long time, lower the retry JobConf conf = new JobConf(); - conf.set(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, "3"); + conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, 2); TestJobClientGetJob client = new TestJobClientGetJob(conf); - JobID id = new JobID("ajob",1); + JobID id = new JobID("ajob", 1); RunningJob rj = mock(RunningJob.class); client.setRunningJob(rj); @@ -236,13 +236,35 @@ public class JobClientUnitTest { assertNotNull(client.getJob(id)); assertEquals(client.getLastGetJobRetriesCounter(), 0); - //3 retry + //2 retries + client.setGetJobRetries(2); + assertNotNull(client.getJob(id)); + assertEquals(client.getLastGetJobRetriesCounter(), 2); + + //beyond yarn.app.mapreduce.client.job.max-retries, will get null client.setGetJobRetries(3); + assertNull(client.getJob(id)); + } + + @Test + public void testGetJobRetryDefault() throws Exception { + + //To prevent the test from running for a very long time, lower the retry + JobConf conf = new JobConf(); + + TestJobClientGetJob client = new TestJobClientGetJob(conf); + JobID id = new JobID("ajob", 1); + RunningJob rj = mock(RunningJob.class); + client.setRunningJob(rj); + + //3 retries (default) + client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES); assertNotNull(client.getJob(id)); - assertEquals(client.getLastGetJobRetriesCounter(), 3); + assertEquals(client.getLastGetJobRetriesCounter(), + MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES); - //beyond MAPREDUCE_JOBCLIENT_GETJOB_MAX_RETRY_KEY, will get null - client.setGetJobRetries(5); + //beyond yarn.app.mapreduce.client.job.max-retries, will get null + client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES + 1); assertNull(client.getJob(id)); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org