tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject git commit: TEZ-1646. Add support for augmenting classpath via configs. (hitesh)
Date Fri, 10 Oct 2014 22:50:48 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 c22c2c3d0 -> a2351fc57


TEZ-1646. Add support for augmenting classpath via configs. (hitesh)

(cherry picked from commit 6d856f9521960c782d4b8c43c14fec16a1d6552c)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a2351fc5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a2351fc5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a2351fc5

Branch: refs/heads/branch-0.5
Commit: a2351fc5742f791129e0ac0b4d1f69889fa3da7f
Parents: c22c2c3
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Oct 10 15:49:39 2014 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Oct 10 15:50:34 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  5 +-
 .../org/apache/tez/common/TezYARNUtils.java     | 58 ++++++++++++-------
 .../apache/tez/dag/api/TezConfiguration.java    | 10 ++++
 .../org/apache/tez/common/TestTezYARNUtils.java | 60 ++++++++++++++++++++
 4 files changed, 109 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/a2351fc5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c5c75c8..3ad32ec 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -13,6 +13,7 @@ ALL CHANGES:
   TEZ-1614. Use setFromConfiguration() in SortMergeJoinExample to demonstrate the usage
   TEZ-1641. Add debug logs in VertexManager to help debugging custom VertexManagerPlugins
   TEZ-1645. Add support for specifying additional local resources via config.
+  TEZ-1646. Add support for augmenting classpath via configs.
 
 Release 0.5.1: 2014-10-02
 
@@ -36,12 +37,12 @@ ALL CHANGES
   TEZ-1580. Change TestOrderedWordCount to optionally use MR configs.
   TEZ-1524. Resolve user group information only if ACLs are enabled.
   TEZ-1581. GroupByOrderByMRRTest no longer functional.
-  TEZ-1157. Optimize broadcast shuffle to download data only once per host. 
+  TEZ-1157. Optimize broadcast shuffle to download data only once per host.
   TEZ-1607. support mr envs in mrrsleep and testorderedwordcount
   TEZ-1499. Add SortMergeJoinExample to tez-examples
   TEZ-1613. Decrease running time for TestAMRecovery
   TEZ-1240. Add system test for propagation of diagnostics for errors
-  TEZ-1618. LocalTaskSchedulerService.getTotalResources() and getAvailableResources() can
get 
+  TEZ-1618. LocalTaskSchedulerService.getTotalResources() and getAvailableResources() can
get
   negative if JVM memory is larger than 2GB
   TEZ-1611. Change DataSource/Sink to be able to supply URIs for credentials
   TEZ-1592. Vertex should wait for all initializers to finish before moving to INITED state

http://git-wip-us.apache.org/repos/asf/tez/blob/a2351fc5/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index de82e8a..74e21c0 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -39,28 +39,40 @@ public class TezYARNUtils {
   private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex());
 
   public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
-    Map<String, String> environment = new HashMap<String, String>();
+    StringBuilder classpathBuilder = new StringBuilder();
 
-    TezYARNUtils.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$(),
-        File.pathSeparator);
+    // Add any additional user-specified classpath
+    String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
+    if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
+      classpathBuilder.append(additionalClasspath)
+          .append(File.pathSeparator);
+    }
 
-    TezYARNUtils.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$() + File.separator + "*",
-        File.pathSeparator);
+    // Add PWD:PWD/*
+    classpathBuilder.append(Environment.PWD.$())
+        .append(File.pathSeparator)
+        .append(Environment.PWD.$() + File.separator + "*")
+        .append(File.pathSeparator);
 
     // Next add the tez libs, if specified via an archive.
     if (usingArchive) {
-      TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          Environment.PWD.$() + File.separator +
-              TezConstants.TEZ_TAR_LR_NAME + File.separator + "*", File.pathSeparator);
-
-      TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          Environment.PWD.$() + File.separator +
-              TezConstants.TEZ_TAR_LR_NAME + File.separator + "lib" + File.separator + "*",
-          File.pathSeparator);
+      // Add PWD/tezlib/*
+      classpathBuilder.append(Environment.PWD.$())
+          .append(File.separator)
+          .append(TezConstants.TEZ_TAR_LR_NAME)
+          .append(File.separator)
+          .append("*")
+          .append(File.pathSeparator);
+
+      // Add PWD/tezlib/lib/*
+      classpathBuilder.append(Environment.PWD.$())
+          .append(File.separator)
+          .append(TezConstants.TEZ_TAR_LR_NAME)
+          .append(File.separator)
+          .append("lib")
+          .append(File.separator)
+          .append("*")
+          .append(File.pathSeparator);
     }
 
     // Last add HADOOP_CLASSPATH, if it's required.
@@ -69,15 +81,17 @@ public class TezYARNUtils {
       for (String c : conf.getStrings(
           YarnConfiguration.YARN_APPLICATION_CLASSPATH,
           YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
-        TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
-            c.trim(), File.pathSeparator);
+        classpathBuilder.append(c.trim())
+            .append(File.pathSeparator);
       }
     } else {
       // Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required.
-      TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          Environment.HADOOP_CONF_DIR.$(), File.pathSeparator);
+      classpathBuilder.append(Environment.HADOOP_CONF_DIR.$())
+          .append(File.pathSeparator);
     }
-    return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name()));
+
+    String classpath = classpathBuilder.toString();
+    return StringInterner.weakIntern(classpath);
   }
 
   public static void appendToEnvFromInputString(Map<String, String> env,

http://git-wip-us.apache.org/repos/asf/tez/blob/a2351fc5/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 06fb3b1..23e4177 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -576,6 +576,16 @@ public class TezConfiguration extends Configuration {
   public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false;
 
   /**
+   * String value.
+   *
+   * Specify additional classpath information to be used for Tez AM and all containers.
+   * This will be prepended to the classpath before all framework specific components have
been
+   * specified.
+   */
+  public static final String TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX =
+      TEZ_PREFIX + "cluster.additional.classpath.prefix";
+
+  /**
    * Session-related properties
    */
   @Private

http://git-wip-us.apache.org/repos/asf/tez/blob/a2351fc5/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
new file mode 100644
index 0000000..5edcdcd
--- /dev/null
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestTezYARNUtils {
+
+  @Test
+  public void testAuxClasspath() {
+    Configuration conf = new Configuration(false);
+    conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar");
+    String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
+    Assert.assertTrue(classpath.contains("foobar"));
+    Assert.assertTrue(classpath.indexOf("foobar") <
+        classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
+    Assert.assertTrue(classpath.indexOf("foobar") <
+        classpath.indexOf(Environment.PWD.$()));
+  }
+
+  @Test
+  public void testBasicArchiveClasspath() {
+    Configuration conf = new Configuration(false);
+    String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
+    Assert.assertTrue(classpath.contains(Environment.PWD.$()));
+    Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*"));
+    Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator +
"*"));
+    Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
+        + "lib" + File.separator + "*"));
+    Assert.assertTrue(classpath.contains(Environment.HADOOP_CONF_DIR.$()));
+    Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
+        classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
+    Assert.assertTrue(classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME) <
+        classpath.indexOf(Environment.HADOOP_CONF_DIR.$()));
+  }
+
+}


Mime
View raw message