tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely (jeagles)
Date Thu, 12 Nov 2015 19:50:53 GMT
Repository: tez
Updated Branches:
  refs/heads/master 3a1dcb6a3 -> bbcafcb81


TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely (jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bbcafcb8
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bbcafcb8
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bbcafcb8

Branch: refs/heads/master
Commit: bbcafcb81a548e1e3bca301e9b1d328c97e36384
Parents: 3a1dcb6
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Thu Nov 12 13:50:20 2015 -0600
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Thu Nov 12 13:50:20 2015 -0600

----------------------------------------------------------------------
 .../dag/library/vertexmanager/ShuffleVertexManager.java  |  6 +++---
 .../library/vertexmanager/TestShuffleVertexManager.java  | 11 ++++++++++-
 2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/bbcafcb8/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index d9c4941..c5016aa 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -949,12 +949,12 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
             ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
             ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);
 
-    if (slowStartMinSrcCompletionFraction < 0
+    if (slowStartMinSrcCompletionFraction < 0 || slowStartMaxSrcCompletionFraction >
1
         || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
       throw new IllegalArgumentException(
           "Invalid values for slowStartMinSrcCompletionFraction"
-              + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and "
-              + "max cannot be < min.");
+              + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be
> 1,"
+              + " and max cannot be < min.");
     }
 
     enableAutoParallelism = conf

http://git-wip-us.apache.org/repos/asf/tez/blob/bbcafcb8/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
index 18d4bc1..fec08b2 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
@@ -589,7 +589,16 @@ public class TestShuffleVertexManager {
       Assert.assertTrue(e.getMessage().contains(
           "Invalid values for slowStartMinSrcCompletionFraction"));
     }
-    
+
+    try {
+      // source vertex have some tasks. max > 1.
+      manager = createManager(conf, mockContext, 0.0f, 95.0f);
+      Assert.assertTrue(false); // should not come here
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "Invalid values for slowStartMinSrcCompletionFraction"));
+    }
+
     try {
       // source vertex have some tasks. min > max
       manager = createManager(conf, mockContext, 0.5f, 0.3f);


Mime
View raw message