hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rkan...@apache.org
Subject hadoop git commit: MAPREDUCE-6776. yarn.app.mapreduce.client.job.max-retries should have a more useful default (miklos.szegedi@cloudera.com via rkanter)
Date Fri, 07 Oct 2016 21:49:27 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b9761f2fc -> 48ca6be76


MAPREDUCE-6776. yarn.app.mapreduce.client.job.max-retries should have a more useful default
(miklos.szegedi@cloudera.com via rkanter)

(cherry picked from commit f3f37e6fb8172f6434e06eb9a137c0c155b3952e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48ca6be7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48ca6be7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48ca6be7

Branch: refs/heads/branch-2
Commit: 48ca6be76e711554631d9babb2ebf74e5adc0f50
Parents: b9761f2
Author: Robert Kanter <rkanter@apache.org>
Authored: Fri Oct 7 14:47:06 2016 -0700
Committer: Robert Kanter <rkanter@apache.org>
Committed: Fri Oct 7 14:47:23 2016 -0700

----------------------------------------------------------------------
 .../apache/hadoop/mapreduce/MRJobConfig.java    |  2 +-
 .../src/main/resources/mapred-default.xml       | 10 +++---
 .../apache/hadoop/mapred/JobClientUnitTest.java | 34 ++++++++++++++++----
 3 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/48ca6be7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 60d357f..e880670 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -502,7 +502,7 @@ public interface MRJobConfig {
    */
   public static final String MR_CLIENT_JOB_MAX_RETRIES =
       MR_PREFIX + "client.job.max-retries";
-  public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 0;
+  public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 3;
 
   /**
    * How long to wait between jobclient retries on failure

http://git-wip-us.apache.org/repos/asf/hadoop/blob/48ca6be7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 4669a2e..84f054e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1502,12 +1502,12 @@
 
 <property>
   <name>yarn.app.mapreduce.client.job.max-retries</name>
-  <value>0</value>
+  <value>3</value>
   <description>The number of retries the client will make for getJob and
-  dependent calls.  The default is 0 as this is generally only needed for
-  non-HDFS DFS where additional, high level retries are required to avoid
-  spurious failures during the getJob call.  30 is a good value for
-  WASB</description>
+    dependent calls.
+    This is needed for non-HDFS DFS where additional, high level
+    retries are required to avoid spurious failures during the getJob call.
+    30 is a good value for WASB</description>
 </property>
 
 <property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/48ca6be7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
index 84b76bf..b5edf2d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
@@ -225,10 +225,10 @@ public class JobClientUnitTest {
 
     //To prevent the test from running for a very long time, lower the retry
     JobConf conf = new JobConf();
-    conf.set(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, "3");
+    conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, 2);
 
     TestJobClientGetJob client = new TestJobClientGetJob(conf);
-    JobID id = new JobID("ajob",1);
+    JobID id = new JobID("ajob", 1);
     RunningJob rj = mock(RunningJob.class);
     client.setRunningJob(rj);
 
@@ -236,13 +236,35 @@ public class JobClientUnitTest {
     assertNotNull(client.getJob(id));
     assertEquals(client.getLastGetJobRetriesCounter(), 0);
 
-    //3 retry
+    //2 retries
+    client.setGetJobRetries(2);
+    assertNotNull(client.getJob(id));
+    assertEquals(client.getLastGetJobRetriesCounter(), 2);
+
+    //beyond yarn.app.mapreduce.client.job.max-retries, will get null
     client.setGetJobRetries(3);
+    assertNull(client.getJob(id));
+  }
+
+  @Test
+  public void testGetJobRetryDefault() throws Exception {
+
+    //To prevent the test from running for a very long time, lower the retry
+    JobConf conf = new JobConf();
+
+    TestJobClientGetJob client = new TestJobClientGetJob(conf);
+    JobID id = new JobID("ajob", 1);
+    RunningJob rj = mock(RunningJob.class);
+    client.setRunningJob(rj);
+
+    //3 retries (default)
+    client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES);
     assertNotNull(client.getJob(id));
-    assertEquals(client.getLastGetJobRetriesCounter(), 3);
+    assertEquals(client.getLastGetJobRetriesCounter(),
+        MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES);
 
-    //beyond MAPREDUCE_JOBCLIENT_GETJOB_MAX_RETRY_KEY, will get null
-    client.setGetJobRetries(5);
+    //beyond yarn.app.mapreduce.client.job.max-retries, will get null
+    client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES + 1);
     assertNull(client.getJob(id));
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message