tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhiyu...@apache.org
Subject [43/50] [abbrv] tez git commit: TEZ-3659. AM/Task classpath should not contain hadoop conf directory. (harishjp)
Date Tue, 28 Mar 2017 19:48:24 GMT
TEZ-3659. AM/Task classpath should not contain hadoop conf directory. (harishjp)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/293c5933
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/293c5933
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/293c5933

Branch: refs/heads/TEZ-1190
Commit: 293c5933f27be96091320f4a5deb80815db33cfd
Parents: a339d05
Author: Harish JP <harishjp@gmail.com>
Authored: Fri Mar 17 08:06:41 2017 +0530
Committer: Harish JP <harishjp@gmail.com>
Committed: Fri Mar 17 08:07:55 2017 +0530

----------------------------------------------------------------------
 .../java/org/apache/tez/common/TezYARNUtils.java     |  7 +++----
 .../org/apache/tez/dag/api/TezConfiguration.java     | 13 +++++++++++++
 .../java/org/apache/tez/common/TestTezYARNUtils.java | 15 +++++++++++++++
 3 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index d5da672..bd6de11 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -18,7 +18,6 @@
 package org.apache.tez.common;
 
 import java.io.File;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -103,10 +102,10 @@ public class TezYARNUtils {
         classpathBuilder.append(c.trim())
             .append(File.pathSeparator);
       }
-    } else {
+    } else if (conf.getBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF,
+        TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT)) {
       // Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required.
-      classpathBuilder.append(Environment.HADOOP_CONF_DIR.$())
-          .append(File.pathSeparator);
+      classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()).append(File.pathSeparator);
     }
 
     if (!userClassesTakesPrecedence) {

http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 5900f5e..40f84e6 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1158,6 +1158,19 @@ public class TezConfiguration extends Configuration {
       TEZ_PREFIX + "cluster.additional.classpath.prefix";
 
   /**
+   * Boolean value.
+   * If this value is true then tez explicitly adds hadoop conf directory into classpath
for AM and
+   * task containers. Default is false.
+   */
+  @Private
+  @Unstable
+  @ConfigurationScope(Scope.CLIENT)
+  @ConfigurationProperty(type="boolean")
+  public static final String TEZ_CLASSPATH_ADD_HADOOP_CONF = TEZ_PREFIX +
+      "classpath.add-hadoop-conf";
+  public static final boolean TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT = false;
+
+  /**
    * Session-related properties
    */
   @Private

http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 9f02801..4fdd61b 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -70,6 +70,21 @@ public class TestTezYARNUtils {
     Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator +
"*"));
     Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
         + "lib" + File.separator + "*"));
+    Assert.assertTrue(!classpath.contains(Environment.HADOOP_CONF_DIR.$()));
+    Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
+        classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
+  }
+
+  @Test(timeout = 5000)
+  public void testNoHadoopConfInClasspath() {
+    Configuration conf = new Configuration(false);
+    conf.setBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, true);
+    String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
+    Assert.assertTrue(classpath.contains(Environment.PWD.$()));
+    Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*"));
+    Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator +
"*"));
+    Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
+        + "lib" + File.separator + "*"));
     Assert.assertTrue(classpath.contains(Environment.HADOOP_CONF_DIR.$()));
     Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
         classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));


Mime
View raw message