tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject tez git commit: TEZ-3466. Tez classpath building to mimic mapreduce classpath building (jlowe)
Date Thu, 13 Oct 2016 22:22:50 GMT
Repository: tez
Updated Branches:
  refs/heads/master b4be3619a -> 43f7b5e3a


TEZ-3466. Tez classpath building to mimic mapreduce classpath building (jlowe)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/43f7b5e3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/43f7b5e3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/43f7b5e3

Branch: refs/heads/master
Commit: 43f7b5e3a6b85e3e4fbb86c3bed81e5b586c79dd
Parents: b4be361
Author: Jason Lowe <jlowe@apache.org>
Authored: Thu Oct 13 22:22:07 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Thu Oct 13 22:22:07 2016 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/common/TezYARNUtils.java     | 36 +++++++++++++-------
 .../apache/tez/dag/api/TezConfiguration.java    | 15 ++++++--
 .../org/apache/tez/common/TestTezYARNUtils.java | 13 +++++++
 .../tez/mapreduce/hadoop/DeprecatedKeys.java    |  4 +++
 .../mapreduce/hadoop/TestDeprecatedKeys.java    |  6 ++++
 6 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 79b0e38..ef6b890 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3466. Tez classpath building to mimic mapreduce classpath building.
   TEZ-3453. Correct the downloaded ATS dag data location for analyzer.
   TEZ-3449. Fix Spelling typos.
   TEZ-3464. Fix findbugs warnings in tez-dag mainLoop

http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index c505ca8..145f928 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -43,20 +43,13 @@ public class TezYARNUtils {
 
   public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
     StringBuilder classpathBuilder = new StringBuilder();
-
-    // Add any additional user-specified classpath
-    String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
-    if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
-      classpathBuilder.append(additionalClasspath)
-          .append(File.pathSeparator);
+    boolean userClassesTakesPrecedence =
+      conf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST,
+          TezConfiguration.TEZ_USER_CLASSPATH_FIRST_DEFAULT);
+    if (userClassesTakesPrecedence) {
+      addUserSpecifiedClasspath(classpathBuilder, conf);
     }
 
-    // Add PWD:PWD/*
-    classpathBuilder.append(Environment.PWD.$())
-        .append(File.pathSeparator)
-        .append(Environment.PWD.$() + File.separator + "*")
-        .append(File.pathSeparator);
-
     String [] tezLibUrisClassPath = conf.getStrings(TezConfiguration.TEZ_LIB_URIS_CLASSPATH);
 
     if(!conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false) &&
@@ -107,10 +100,29 @@ public class TezYARNUtils {
           .append(File.pathSeparator);
     }
 
+    if (!userClassesTakesPrecedence) {
+      addUserSpecifiedClasspath(classpathBuilder, conf);
+    }
     String classpath = classpathBuilder.toString();
     return StringInterner.weakIntern(classpath);
   }
 
+  private static void addUserSpecifiedClasspath(StringBuilder classpathBuilder,
+      Configuration conf) {
+    // Add any additional user-specified classpath
+    String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
+    if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
+      classpathBuilder.append(additionalClasspath)
+          .append(File.pathSeparator);
+    }
+
+    // Add PWD:PWD/*
+    classpathBuilder.append(Environment.PWD.$())
+        .append(File.pathSeparator)
+        .append(Environment.PWD.$() + File.separator + "*")
+        .append(File.pathSeparator);
+  }
+
   public static void appendToEnvFromInputString(Map<String, String> env,
       String envString, String classPathSeparator) {
     if (envString != null && envString.length() > 0) {

http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 5aa8f7e..77ea4ff 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1094,11 +1094,22 @@ public class TezConfiguration extends Configuration {
   public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false;
 
   /**
+   * Boolean value.
+   * Specify whether the user classpath takes precedence over the Tez framework
+   * classpath.
+   */
+  @ConfigurationScope(Scope.CLIENT)
+  @ConfigurationProperty(type="boolean")
+  public static final String TEZ_USER_CLASSPATH_FIRST = TEZ_PREFIX + "user.classpath.first";
+  public static final boolean TEZ_USER_CLASSPATH_FIRST_DEFAULT = true;
+
+  /**
    * String value.
    *
    * Specify additional classpath information to be used for Tez AM and all containers.
-   * This will be prepended to the classpath before all framework specific components have
been
-   * specified.
+   * If {@link #TEZ_USER_CLASSPATH_FIRST} is true then this will be added to the classpath
+   * before all framework specific components have been specified, otherwise this will
+   * be added after the framework specific components.
    */
   @ConfigurationScope(Scope.AM)
   @ConfigurationProperty

http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 2dabf51..092bd50 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -43,6 +43,19 @@ public class TestTezYARNUtils {
         classpath.indexOf(Environment.PWD.$()));
   }
 
+  @Test(timeout = 20000)
+  public void testUserClasspathFirstFalse() {
+    Configuration conf = new Configuration(false);
+    conf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false);
+    conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar");
+    String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
+    Assert.assertTrue(classpath.contains("foobar"));
+    Assert.assertTrue(classpath.indexOf("foobar") >
+        classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
+    Assert.assertTrue(classpath.indexOf("foobar") >
+        classpath.indexOf(Environment.PWD.$()));
+  }
+
   @Test(timeout = 5000)
   public void testBasicArchiveClasspath() {
     Configuration conf = new Configuration(false);

http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index a826f6d..9ae58c0 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -86,6 +86,8 @@ public class DeprecatedKeys {
         TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS);
     mrParamToDAGParamMap.put(MRJobConfig.JOB_TAGS,
         TezConfiguration.TEZ_APPLICATION_TAGS);
+    mrParamToDAGParamMap.put(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST,
+        TezConfiguration.TEZ_USER_CLASSPATH_FIRST);
   }
 
   // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current
initialization in MRAppMaster, TezChild.
@@ -157,6 +159,8 @@ public class DeprecatedKeys {
     registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
 
     registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
+
+    registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, TezConfiguration.TEZ_USER_CLASSPATH_FIRST);
   }
   
   private static void addDeprecatedKeys() {

http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index ae218a0..c574e38 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -43,6 +43,7 @@ public class TestDeprecatedKeys {
     jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f);
     jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true);
     jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f);
+    jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
 
     MRHelpers.translateMRConfToTez(jobConf);
 
@@ -64,6 +65,7 @@ public class TestDeprecatedKeys {
     assertEquals(0.33f,
         jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT,
0),
         0.01f);
+    assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true));
   }
 
   @Test(timeout = 5000)
@@ -76,6 +78,7 @@ public class TestDeprecatedKeys {
     jobConf.setInt(MRJobConfig.IO_SORT_MB, 100);
     jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100);
     jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f);
+    jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
 
     jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
     jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200);
@@ -100,6 +103,7 @@ public class TestDeprecatedKeys {
     jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
     jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
     jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
+    jobConf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false);
 
     jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
     jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true);
@@ -131,6 +135,7 @@ public class TestDeprecatedKeys {
     assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS,
""));
     assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false));
     assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION,
0.0f), 0.0f);
+    assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true));
 
     assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD));
     assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES));
@@ -156,6 +161,7 @@ public class TestDeprecatedKeys {
     assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
     assertNull(jobConf.get("map.sort.class"));
     assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART));
+    assertNull(jobConf.get(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST));
   }
 
 }


Mime
View raw message