Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 54CCB200C45 for ; Tue, 28 Mar 2017 21:47:48 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5359D160B6B; Tue, 28 Mar 2017 19:47:48 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 07788160B89 for ; Tue, 28 Mar 2017 21:47:44 +0200 (CEST) Received: (qmail 72822 invoked by uid 500); 28 Mar 2017 19:47:44 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 71365 invoked by uid 99); 28 Mar 2017 19:47:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 28 Mar 2017 19:47:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 66036E0779; Tue, 28 Mar 2017 19:47:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhiyuany@apache.org To: commits@tez.apache.org Date: Tue, 28 Mar 2017 19:48:24 -0000 Message-Id: <1ecbbb2850ed46858c311451a4a34490@git.apache.org> In-Reply-To: <3c41745652fc4012b06b29e1b1cbf344@git.apache.org> References: <3c41745652fc4012b06b29e1b1cbf344@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [43/50] [abbrv] tez git commit: TEZ-3659. AM/Task classpath should not contain hadoop conf directory. (harishjp) archived-at: Tue, 28 Mar 2017 19:47:48 -0000 TEZ-3659. AM/Task classpath should not contain hadoop conf directory. (harishjp) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/293c5933 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/293c5933 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/293c5933 Branch: refs/heads/TEZ-1190 Commit: 293c5933f27be96091320f4a5deb80815db33cfd Parents: a339d05 Author: Harish JP Authored: Fri Mar 17 08:06:41 2017 +0530 Committer: Harish JP Committed: Fri Mar 17 08:07:55 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/tez/common/TezYARNUtils.java | 7 +++---- .../org/apache/tez/dag/api/TezConfiguration.java | 13 +++++++++++++ .../java/org/apache/tez/common/TestTezYARNUtils.java | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java index d5da672..bd6de11 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java @@ -18,7 +18,6 @@ package org.apache.tez.common; import java.io.File; -import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -103,10 +102,10 @@ public class TezYARNUtils { classpathBuilder.append(c.trim()) .append(File.pathSeparator); } - } else { + } else if (conf.getBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, + TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT)) { // Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required. - classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()) - .append(File.pathSeparator); + classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()).append(File.pathSeparator); } if (!userClassesTakesPrecedence) { http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 5900f5e..40f84e6 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -1158,6 +1158,19 @@ public class TezConfiguration extends Configuration { TEZ_PREFIX + "cluster.additional.classpath.prefix"; /** + * Boolean value. + * If this value is true then tez explicitly adds hadoop conf directory into classpath for AM and + * task containers. Default is false. + */ + @Private + @Unstable + @ConfigurationScope(Scope.CLIENT) + @ConfigurationProperty(type="boolean") + public static final String TEZ_CLASSPATH_ADD_HADOOP_CONF = TEZ_PREFIX + + "classpath.add-hadoop-conf"; + public static final boolean TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT = false; + + /** * Session-related properties */ @Private http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java index 9f02801..4fdd61b 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java @@ -70,6 +70,21 @@ public class TestTezYARNUtils { Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*")); Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "lib" + File.separator + "*")); + Assert.assertTrue(!classpath.contains(Environment.HADOOP_CONF_DIR.$())); + Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) < + classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME)); + } + + @Test(timeout = 5000) + public void testNoHadoopConfInClasspath() { + Configuration conf = new Configuration(false); + conf.setBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, true); + String classpath = TezYARNUtils.getFrameworkClasspath(conf, true); + Assert.assertTrue(classpath.contains(Environment.PWD.$())); + Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*")); + Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*")); + Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + + "lib" + File.separator + "*")); Assert.assertTrue(classpath.contains(Environment.HADOOP_CONF_DIR.$())); Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) < classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));