tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject [3/9] tez git commit: TEZ-3131. Support a way to override test_root_dir for FaultToleranceTestRunner. (hitesh)
Date Wed, 24 Feb 2016 18:51:37 GMT
TEZ-3131. Support a way to override test_root_dir for FaultToleranceTestRunner. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/44ca2295
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/44ca2295
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/44ca2295

Branch: refs/heads/TEZ-2980
Commit: 44ca2295f8209dd83041e426d0dca95721461de6
Parents: f38e23c
Author: Hitesh Shah <hitesh@apache.org>
Authored: Mon Feb 22 17:13:16 2016 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Mon Feb 22 17:13:16 2016 -0800

----------------------------------------------------------------------
 CHANGES.txt                                          |  1 +
 .../library/vertexmanager/ShuffleVertexManager.java  |  4 +++-
 .../apache/tez/test/FaultToleranceTestRunner.java    | 15 ++++++++++++---
 3 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dc27f93..2311c55 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
   TEZ-3029. Add an onError method to service plugin contexts.
 
 ALL CHANGES:
+  TEZ-3131. Support a way to override test_root_dir for FaultToleranceTestRunner.
   TEZ-3067. Links to tez configs documentation should be bubbled up to top-level release
page.
   TEZ-3123. Containers can get re-used even with conflicting local resources.
   TEZ-3117. Deadlock in Edge and Vertex code

http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index 410ad73..ea00532 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -1004,7 +1004,9 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
       throw new IllegalArgumentException(
           "Invalid values for slowStartMinSrcCompletionFraction"
               + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be
> 1,"
-              + " and max cannot be < min.");
+              + " and max cannot be < min."
+              + ", configuredMin=" + slowStartMinSrcCompletionFraction
+              + ", configuredMax=" + slowStartMaxSrcCompletionFraction);
     }
 
     enableAutoParallelism = conf

http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
index 84e5b97..94242a7 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.util.GenericOptionsParser;
  */
 public class FaultToleranceTestRunner {
   
-  static String TEST_ROOT_DIR = "tmp";
+  static String DEFAULT_FT_STAGING_DIR = "tmp";
+  static String FT_STAGING_DIR = "tez.test-fault-tolerance.staging-dir";
   Configuration conf = null;
   TezClient tezSession = null;
   Resource defaultResource = Resource.newInstance(100, 0);
@@ -57,8 +58,16 @@ public class FaultToleranceTestRunner {
        tezConf = new TezConfiguration(new YarnConfiguration(this.conf));
     }
     FileSystem defaultFs = FileSystem.get(tezConf);
-    
-    Path remoteStagingDir = defaultFs.makeQualified(new Path(TEST_ROOT_DIR, String
+
+    Path testRootDir;
+    if (tezConf.get(FT_STAGING_DIR) != null
+        && !tezConf.get(FT_STAGING_DIR).isEmpty()) {
+      testRootDir = new Path(tezConf.get(FT_STAGING_DIR));
+    } else {
+      testRootDir = new Path(DEFAULT_FT_STAGING_DIR);
+    }
+
+    Path remoteStagingDir = defaultFs.makeQualified(new Path(testRootDir, String
         .valueOf(new Random().nextInt(100000))));
     TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir);
     


Mime
View raw message