tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject [1/2] tez git commit: TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely (jeagles)
Date Thu, 12 Nov 2015 19:57:04 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 aacad1866 -> 7121c3aed


TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely (jeagles)

(cherry picked from commit bbcafcb81a548e1e3bca301e9b1d328c97e36384)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0a9bad95
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0a9bad95
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0a9bad95

Branch: refs/heads/branch-0.7
Commit: 0a9bad95cd3069f40b1d29201883af48ac52256f
Parents: aacad18
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Thu Nov 12 13:50:20 2015 -0600
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Thu Nov 12 13:53:53 2015 -0600

----------------------------------------------------------------------
 .../dag/library/vertexmanager/ShuffleVertexManager.java  |  6 +++---
 .../library/vertexmanager/TestShuffleVertexManager.java  | 11 ++++++++++-
 2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/0a9bad95/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index d177460..4dfd411 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -790,12 +790,12 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
             ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
             ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);
 
-    if (slowStartMinSrcCompletionFraction < 0
+    if (slowStartMinSrcCompletionFraction < 0 || slowStartMaxSrcCompletionFraction >
1
         || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
       throw new IllegalArgumentException(
           "Invalid values for slowStartMinSrcCompletionFraction"
-              + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and "
-              + "max cannot be < min.");
+              + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be
> 1,"
+              + " and max cannot be < min.");
     }
 
     enableAutoParallelism = conf

http://git-wip-us.apache.org/repos/asf/tez/blob/0a9bad95/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
index 8807674..4f93049 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
@@ -535,7 +535,16 @@ public class TestShuffleVertexManager {
       Assert.assertTrue(e.getMessage().contains(
           "Invalid values for slowStartMinSrcCompletionFraction"));
     }
-    
+
+    try {
+      // source vertex have some tasks. max > 1.
+      manager = createManager(conf, mockContext, 0.0f, 95.0f);
+      Assert.assertTrue(false); // should not come here
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "Invalid values for slowStartMinSrcCompletionFraction"));
+    }
+
     try {
       // source vertex have some tasks. min > max
       manager = createManager(conf, mockContext, 0.5f, 0.3f);


Mime
View raw message