tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-2966. Tez does not honor mapreduce.task.timeout (jeagles)
Date Fri, 11 Dec 2015 16:23:10 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 5861103cf -> 3a32c3505


TEZ-2966. Tez does not honor mapreduce.task.timeout (jeagles)

(cherry picked from commit 460e0b2a01c00748bb80ad2314b2168dfa1e9615)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/3a32c350
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/3a32c350
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/3a32c350

Branch: refs/heads/branch-0.7
Commit: 3a32c3505fbfd2552b28533189067c26786fa18f
Parents: 5861103
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Fri Dec 11 10:16:25 2015 -0600
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Fri Dec 11 10:22:41 2015 -0600

----------------------------------------------------------------------
 CHANGES.txt                                            |  1 +
 .../apache/tez/mapreduce/hadoop/DeprecatedKeys.java    |  3 ++-
 .../tez/mapreduce/hadoop/TestDeprecatedKeys.java       | 13 ++++++++++---
 3 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/3a32c350/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 497a9ab..1d1e137 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,7 @@ INCOMPATIBLE CHANGES
   TEZ-2949. Allow duplicate dag names within session for Tez.
 
 ALL CHANGES
+  TEZ-2966. Tez does not honor mapreduce.task.timeout
   TEZ-2346. TEZ-UI: Lazy load other info / counter data
   TEZ-2973. Backport Analyzers to branch-0.7
   TEZ-2975. Bump up apache commons dependency.

http://git-wip-us.apache.org/repos/asf/tez/blob/3a32c350/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 49f95c0..345f7aa 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -84,7 +84,7 @@ public class DeprecatedKeys {
       TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
   }
 
-  // TODO TEZAM4 Sometime, make sure this gets loaded by default. Insteaf of the current
initialization in MRAppMaster, TezChild.
+  // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current
initialization in MRAppMaster, TezChild.
   // Maybe define in an TEZConfiguration / TEZ JobConf variant.
   
   public static void init() {
@@ -154,6 +154,7 @@ public class DeprecatedKeys {
 
     registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
 
+    registerMRToRuntimeKeyTranslation(MRJobConfig.TASK_TIMEOUT, TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS);
   }
   
   private static void addDeprecatedKeys() {

http://git-wip-us.apache.org/repos/asf/tez/blob/3a32c350/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index 2414743..06d2990 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNull;
 
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager;
 import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
 import org.apache.tez.runtime.library.common.Constants;
@@ -42,6 +43,7 @@ public class TestDeprecatedKeys {
     jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f);
     jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true);
     jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f);
+    jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000);
 
     MRHelpers.translateMRConfToTez(jobConf);
 
@@ -63,11 +65,12 @@ public class TestDeprecatedKeys {
     assertEquals(0.33f,
         jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT,
0),
         0.01f);
+    assertEquals(1000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000));
   }
 
   @Test(timeout = 5000)
   /**
-   * Set of keys that can be overriden at tez runtime
+   * Set of keys that can be overridden at tez runtime
    */
   public void verifyTezOverridenKeys() {
     JobConf jobConf = new JobConf();
@@ -75,7 +78,8 @@ public class TestDeprecatedKeys {
     jobConf.setInt(MRJobConfig.IO_SORT_MB, 100);
     jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100);
     jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f);
-    
+    jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000);
+
     jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
     jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200);
     jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true);
@@ -99,7 +103,8 @@ public class TestDeprecatedKeys {
     jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
     jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
     jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
-    
+    jobConf.setInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000);
+
     jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
     jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true);
 
@@ -130,6 +135,7 @@ public class TestDeprecatedKeys {
     assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS,
""));
     assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false));
     assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION,
0.0f), 0.0f);
+    assertEquals(2000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000));
 
     assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD));
     assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES));
@@ -155,6 +161,7 @@ public class TestDeprecatedKeys {
     assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
     assertNull(jobConf.get("map.sort.class"));
     assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART));
+    assertNull(jobConf.get(MRJobConfig.TASK_TIMEOUT));
   }
 
 }


Mime
View raw message