tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1300. Change default tez classpath to not include hadoop jars from the cluster. (sseth)
Date Mon, 28 Jul 2014 23:25:12 GMT
Repository: tez
Updated Branches:
  refs/heads/master 3e785f183 -> cb3338e86


TEZ-1300. Change default tez classpath to not include hadoop jars from
the cluster. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cb3338e8
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cb3338e8
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cb3338e8

Branch: refs/heads/master
Commit: cb3338e86e88ae23bb6ec11945c3694d2277eff0
Parents: 3e785f1
Author: Siddharth Seth <sseth@apache.org>
Authored: Mon Jul 28 16:24:18 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Mon Jul 28 16:24:18 2014 -0700

----------------------------------------------------------------------
 BUILDING.txt                                    |  12 +-
 CHANGES.txt                                     |   2 +
 INSTALL.txt                                     |  58 +++++----
 docs/src/site/apt/install.apt                   |   6 +-
 docs/src/site/apt/install_0_5_0.apt             | 118 ++++++++++++++++++
 .../java/org/apache/tez/client/TezClient.java   |  14 +--
 .../org/apache/tez/client/TezClientUtils.java   | 120 ++++++++++++-------
 .../org/apache/tez/common/TezYARNUtils.java     |  36 ++++--
 .../apache/tez/dag/api/TezConfiguration.java    |  30 ++++-
 .../org/apache/tez/dag/api/TezConstants.java    |   5 +
 tez-dist/pom.xml                                |  23 +---
 tez-dist/src/main/assembly/tez-dist-full.xml    |  44 -------
 tez-dist/src/main/assembly/tez-dist-partial.xml |  51 ++++++++
 tez-dist/src/main/assembly/tez-dist.xml         |   7 +-
 .../org/apache/tez/test/MiniTezCluster.java     |   2 +
 15 files changed, 370 insertions(+), 158 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 80a824d..14753b9 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -41,12 +41,12 @@ Maven build goals:
  * Run clover                : mvn test -Pclover [-Dclover.license=${user.home}/clover.license]
  * Run Rat                   : mvn apache-rat:check
  * Build javadocs            : mvn javadoc:javadoc
- * Build distribution        : mvn package[-Dtar][-Dhadoop.version=2.4.0]
+ * Build distribution        : mvn package[-Dhadoop.version=2.4.0]
  * Visualize state machines  : mvn compile -Pvisualize -DskipTests=true
  
 Build options:
  
- * Use -Dtar to create a TAR with the distribution (tar.gz will be created under /tez-dist/target)
+ * Use -Dpackage.format to create distributions with a format other than .tar.gz (mvn-assembly-plugin
formats). 
  * Use -Dclover.license to specify the path to the clover license file
  * Use -Dhadoop.version to specify the version of hadoop to build tez against
  * Use -Dprotoc.path to specify the path to protoc
@@ -67,16 +67,16 @@ Tez runs on top of Apache Hadoop YARN and requires hadoop version 2.2.0
or highe
 By default, it can be compiled against hadoop versions 2.4.0 and higher by just
 specifying the hadoop.version. For example, to build tez against hadoop 3.0.0-SNAPSHOT 
 
- $ mvn package -Dtar -Dhadoop.version=3.0.0-SNAPSHOT
+ $ mvn package -Dhadoop.version=3.0.0-SNAPSHOT
  
 However, to compile against hadoop versions lower than 2.4.0, the hadoop24 profile needs
 to be disabled
 
- $ mvn package -Dtar -Dhadoop.version=2.2.0 -P\!hadoop24
+ $ mvn package  -Dhadoop.version=2.2.0 -P\!hadoop24
 
 To skip Tests and java docs
 
- $ mvn package -Dtar -Dhadoop.version=3.0.0-SNAPSHOT -DskipTests -Dmaven.javadoc.skip=true
+ $ mvn package -Dhadoop.version=3.0.0-SNAPSHOT -DskipTests -Dmaven.javadoc.skip=true
 
 ----------------------------------------------------------------------------------
 Protocol Buffer compiler:
@@ -91,7 +91,7 @@ protoc is looked up in the PATH.
 
 You can also specify the path to protoc while building using -Dprotoc.path
 
- $ mvn package -DskipTests -Dtar -Dprotoc.path=/usr/local/bin/protoc
+ $ mvn package -DskipTests -Dprotoc.path=/usr/local/bin/protoc
 
 
 ----------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 699b291..fef7b8d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -34,6 +34,8 @@ INCOMPATIBLE CHANGES
   (Chen He via bikas)
   TEZ-1311. get sharedobjectregistry from the context instead of a static
   (bikas)"
+  TEZ-1300. Change deploy mechanism for Tez to be based on a tarball which
+  includes Hadoop libs.
 
 Release 0.4.0-incubating: 2014-04-05
 

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/INSTALL.txt
----------------------------------------------------------------------
diff --git a/INSTALL.txt b/INSTALL.txt
index 6b32fe0..1328997 100644
--- a/INSTALL.txt
+++ b/INSTALL.txt
@@ -10,38 +10,37 @@ Install/Deploy Instructions
 ===========================
 
 1) Deploy Apache Hadoop either using the 2.2.0 release or a compatible 2.x version.
-2) Build tez using "mvn clean install -DskipTests=true -Dmaven.javadoc.skip=true"
+2) Build tez using "mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true"
    - If you prefer to run the unit tests, remove skipTests from the command above.
-   - If you would like to create a tarball of the release, use
-     "mvn clean package -Dtar -DskipTests=true -Dmaven.javadoc.skip=true"
-3) Copy the tez jars and their dependencies into HDFS.
-   - The tez jars and dependencies will be found in tez-dist/target/tez-0.5.0-SNAPSHOT/tez-0.5.0-SNAPSHOT
-     if you run the intial command mentioned in step 2.
+   - A tarball containing the libraries required to run tez will be created at tez-dist/target/tez-0.5.0-SNAPSHOT.tar.gz
+3) Copy the relevant tez tarball into HDFS, and configure tez-site.xml
+   - A tez tarball containing tez and hadoop libraries will be found at tez-dist/target/tez-0.5.0-SNAPSHOT.tar.gz
    - Assuming that the tez jars are put in /apps/ on HDFS, the command would be
-     "hadoop dfs -put tez-dist/target/tez-0.5.0-SNAPSHOT/tez-0.5.0-SNAPSHOT /apps/"
-   - Please do not upload the tarball to HDFS, upload only the jars.
-4) Configure tez-site.xml to set tez.lib.uris to point to the paths in HDFS containing
-   the jars. Please note that the paths are not searched recursively so for <basedir>
-   and <basedir>/lib/, you will need to configure the 2 paths as a comma-separated
list.
-   - Assuming you followed step 3, the value would be:
-      "${fs.default.name}/apps/tez-0.5.0-SNAPSHOT,${fs.default.name}/apps/tez-0.5.0-SNAPSHOT/lib/"
-5) Modify mapred-site.xml to change "mapreduce.framework.name" property from its
+     "hadoop dfs -mkdir /apps/tez-0.5.0-SNAPSHOT"
+     "hadoop dfs -copyFromLocal tez-dist/target/tez-0.5.0-SNAPSHOT-archive.tar.gz /apps/tez-0.5.0-SNAPSHOT/"
+   - tez-site.xml configuration 
+     - Set tez.lib.uris to point to the tar.gz uploaded to HDFS. Assuming the steps mentioned
so far were followed,
+       set tez.lib.uris to "${fs.default.name}/apps/tez-0.5.0-SNAPSHOT/tez-0.5.0-SNAPSHOT.tar.gz"
+     - Ensure tez.use.cluster.hadoop-libs is not set in tez-site.xml, or if it is set, the
value should be false
+4) Optional: If running existing MapReduce jobs on Tez. Modify mapred-site.xml to change
"mapreduce.framework.name" property from its
    default value of "yarn" to "yarn-tez"
-6) set HADOOP_CLASSPATH to have the following paths in it:
-   - TEZ_CONF_DIR - location of tez-site.xml
-   - TEZ_JARS and TEZ_JARS/libs - location of the tez jars and dependencies.
-   - The command to set up the classpath should be something like:
-     "export HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*".
-   - Please note the "*" which is an important requirement when setting up classpaths for
directories container jar files.
+5) Configure the client node to include the tez-libraries in the hadoop classpath
+   - Extract the tez tarball created in step 2 to a local directory - (assuming TEZ_JARS
is where the files will be decompressed for the next steps)
+     "tar -xvzf tez-dist/target/tez-0.5.0-SNAPSHOT.tar.gz -C $TEZ_JARS"
+   - set HADOOP_CLASSPATH to include the tez-libraries
+     - set TEZ_CONF_DIR to the location of tez-site.xml
+     - The command to set up the classpath should be something like:
+       "export HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*"
+     - Please note the "*" which is an important requirement when setting up classpaths for
directories containing jar files.
 
-7) Submit a MR job as you normally would using something like:
+6) Submit a MR job as you normally would using something like:
 
 $HADOOP_PREFIX/bin/hadoop jar hadoop-mapreduce-client-jobclient-2.2.0-tests.jar sleep -mt
1 -rt 1 -m 1 -r 1
 
 This will use the TEZ DAG ApplicationMaster to run the MR job. This can be
 verified by looking at the AM's logs from the YARN ResourceManager UI.
 
-8) There is a basic example of using an MRR job in the tez-mapreduce-examples.jar. Refer
to OrderedWordCount.java
+7) There is a basic example of using an MRR job in the tez-mapreduce-examples.jar. Refer
to OrderedWordCount.java
 in the source code. To run this example:
 
 $HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount <input> <output>
@@ -60,3 +59,18 @@ The above will run multiple DAGs for each input-output pair. To use TEZ
sessions
 set -DUSE_TEZ_SESSION=true
 
 $HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount -DUSE_TEZ_SESSION=true
<input1> <output1> <input2> <output2>
+
+
+
+
+
+Alternate machanism to setup Tez to use Hadoop libraries from the cluster.
+Step 3 changes as follows. Also subsequent steps would use tez-dist/target/tez-0.5.0-SNAPSHOT-partial.tar.gz
instead of tez-dist/target/tez-0.5.0-SNAPSHOT.tar.gz
+   - A tez build without Hadoop dependencies will be available at tez-dist/target/tez-0.5.0-SNAPSHOT-partial.tar.gz
+   - Assuming that the tez jars are put in /apps/ on HDFS, the command would be
+     "hadoop dfs -mkdir /apps/tez-0.5.0-SNAPSHOT"
+     "hadoop dfs -copyFromLocal tez-dist/target/tez-0.5.0-SNAPSHOT-archive-partial.tar.gz
/apps/tez-0.5.0-SNAPSHOT"
+   - tez-site.xml configuration
+     - Set tez.lib.uris to point to the paths in HDFS containing the tez jars. Assuming the
steps mentioned so far were followed,
+     set tez.lib.uris to "${fs.default.name}/apps/tez-0.5.0-SNAPSHOT/tez-0.5.0-SNAPSHOT-partial.tar.gz
+     - Also set tez.use.cluster.hadoop-libs to true

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/docs/src/site/apt/install.apt
----------------------------------------------------------------------
diff --git a/docs/src/site/apt/install.apt b/docs/src/site/apt/install.apt
index 471624b..6b4af8a 100644
--- a/docs/src/site/apt/install.apt
+++ b/docs/src/site/apt/install.apt
@@ -17,7 +17,9 @@
   Install and Deployment Instructions
   -----
 
-Install/Deploy Instructions
+{{{./install_0_5_0.html}Install instructions for Tez-0.5.0-SNAPSHOT - master branch}}
+
+Install/Deploy Instructions for the latest Tez release {{{http://www.apache.org/dyn/closer.cgi/incubator/tez/tez-0.4.1-incubating/}(Tez-0.4.1
src)}}
 
   [[i]] Deploy Apache Hadoop using either the 2.2.0 release or a compatible 2.x version.
    
@@ -25,7 +27,7 @@ Install/Deploy Instructions
 
   [[i]] Build tez using "mvn clean install -DskipTests=true -Dmaven.javadoc.skip=true"
 
-    * This assumes that you have already installed JDK6 or later, Maven 3 or later and Protocol
Buffers (protoc compiler) 2.2 or later
+    * This assumes that you have already installed JDK6 or later, Maven 3 or later and Protocol
Buffers (protoc compiler) 2.5 or later
 
     * If you prefer to run the unit tests, remove skipTests from the command above.
 

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/docs/src/site/apt/install_0_5_0.apt
----------------------------------------------------------------------
diff --git a/docs/src/site/apt/install_0_5_0.apt b/docs/src/site/apt/install_0_5_0.apt
new file mode 100644
index 0000000..90a94c2
--- /dev/null
+++ b/docs/src/site/apt/install_0_5_0.apt
@@ -0,0 +1,118 @@
+~~ Licensed to the Apache Software Foundation (ASF) under one or more
+~~ contributor license agreements.  See the NOTICE file distributed with
+~~ this work for additional information regarding copyright ownership.
+~~ The ASF licenses this file to You under the Apache License, Version 2.0
+~~ (the "License"); you may not use this file except in compliance with
+~~ the License.  You may obtain a copy of the License at
+~~
+~~     http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License.
+~~
+  -----
+  Install and Deployment Instructions
+  -----
+
+Install/Deploy Instructions for Tez-current (0.5.0-SNAPSHOT, branch master)
+
+  [[i]] Deploy Apache Hadoop using either the 2.2.0 release or a compatible 2.x version.
+   
+    * One thing to note though when compiling Tez is that you will need to change the value
of the hadoop.version property in the top-level pom.xml to match the version of the hadoop
branch being used.
+
+  [[i]] Build tez using "mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true"
+
+    * This assumes that you have already installed JDK6 or later, Maven 3 or later and Protocol
Buffers (protoc compiler) 2.5 or later
+
+    * If you prefer to run the unit tests, remove skipTests from the command above.
+
+    * If you use Eclipse IDE, you can import the peojects using "Import/Maven/Existing Maven
Projects". Eclipse does not automatically generate Java sources or include the generated sources
into the projects. Please build using maven as described above and then use Project Properties
to include "target/generated-sources/java" as a source directory into the "Java Build Path"
for these projects: tez-api, tez-mapreduce, tez-runtime-internals and tez-runtime-library.
This needs to be done just once after importing the project.
+
+  [[i]] Copy the relevant tez tarball into HDFS, and configure tez-site.xml
+
+    * A tez tarball containing tez and hadoop libraries will be found at tez-dist/target/tez-0.5.0-SNAPSHOT.tar.gz
+
+    * Assuming that the tez jars are put in /apps/ on HDFS, the command would be
+
++--+
+
+    "hadoop dfs -mkdir /apps/tez-0.5.0-SNAPSHOT"
+    "hadoop dfs -copyFromLocal tez-dist/target/tez-0.5.0-SNAPSHOT-archive.tar.gz /apps/tez-0.5.0-SNAPSHOT/"
+
++--+
+
+    * tez-site.xml configuration.
+
+        * Set tez.lib.uris to point to the tar.gz uploaded to HDFS. Assuming the steps mentioned
so far were followed,
+
++--+
+
+        set tez.lib.uris to "${fs.default.name}/apps/tez-0.5.0-SNAPSHOT/tez-0.5.0-SNAPSHOT.tar.gz"
+
++--+
+        * Ensure tez.use.cluster.hadoop-libs is not set in tez-site.xml, or if it is set,
the value should be false
+
+  [[i]] Optional: If running existing MapReduce jobs on Tez. Modify mapred-site.xml to change
"mapreduce.framework.name" property from its default value of "yarn" to "yarn-tez"
+
+  [[i]] Configure the client node to include the tez-libraries in the hadoop classpath
+
+    * Extract the tez tarball created in step 2 to a local directory - (assuming TEZ_JARS
is where the files will be decompressed for the next steps)
+
++--+
+
+"tar -xvzf tez-dist/target/tez-0.5.0-SNAPSHOT.tar.gz -C $TEZ_JARS"
+
++--+
+    * set TEZ_CONF_DIR to the location of tez-site.xml
+
+    * The command to set up the classpath should be something like:
+
++--+
+
+"export HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*"
+
++--+
+
+    * Please note the "*" which is an important requirement when setting up classpaths for
directories containing jar files. 
+
+  [[i]] Submit a MR job as you normally would using something like:
+
++---+
+
+$HADOOP_PREFIX/bin/hadoop jar hadoop-mapreduce-client-jobclient-3.0.0-SNAPSHOT-tests.jar
sleep -mt 1 -rt 1 -m 1 -r 1
+
++---+
+
+    This will use the TEZ DAG ApplicationMaster to run the MR job. This can be verified by
looking at the AM's logs from the YARN ResourceManager UI.
+
+  [[i]] There is a basic example of using an MRR job in the tez-mapreduce-examples.jar. Refer
to OrderedWordCount.java
+in the source code. To run this example:
+
++---+
+
+$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount <input> <output>
+
++---+
+
+    This will use the TEZ DAG ApplicationMaster to run the ordered word count job. This job
is similar to the word count example except that it also orders all words based on the frequency
of occurrence.
+
+    There are multiple variations to run orderedwordcount. You can use it to run multiple
DAGs serially on different inputs/outputs. These DAGs could be run separately as different
applications or serially within a single TEZ session.
+
++---+
+
+$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount <input1>
<output1> <input2> <output2> <input3> <output3> ...
+
++---+
+
+    The above will run multiple DAGs for each input-output pair.
+
+    To use TEZ sessions, set -DUSE_TEZ_SESSION=true
+
++---+
+
+$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount -DUSE_TEZ_SESSION=true
<input1> <output1> <input2> <output2>
+
++---+

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 28546c4..1f29d61 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -18,7 +18,6 @@
 
 package org.apache.tez.client;
 
-import java.io.File;
 import java.io.IOException;
 import java.text.NumberFormat;
 import java.util.Map;
@@ -30,7 +29,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -345,7 +343,8 @@ public class TezClient {
     for (Vertex v : dag.getVertices()) {
       Map<String, String> taskEnv = v.getTaskEnvironment();
       TezYARNUtils.setupDefaultEnv(taskEnv, amConfig.getTezConfiguration(),
-          TezConfiguration.TEZ_TASK_LAUNCH_ENV, TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT);
+          TezConfiguration.TEZ_TASK_LAUNCH_ENV, TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT,
+          TezClientUtils.usingTezLibsFromArchive(getTezJarResources(sessionCredentials)));
       TezClientUtils.setDefaultLaunchCmdOpts(v, amConfig.getTezConfiguration());
     }
     
@@ -535,12 +534,11 @@ public class TezClient {
             + " timeout interval, timeoutSecs=" + clientTimeout);
       }
 
-      String classpath = TezYARNUtils
-        .getFrameworkClasspath(amConfig.getYarnConfiguration());
       Map<String, String> contextEnv = context.getEnvironment();
-      TezYARNUtils.addToEnvironment(contextEnv,
-        ApplicationConstants.Environment.CLASSPATH.name(),
-        classpath, File.pathSeparator);
+      TezYARNUtils.setupDefaultEnv(contextEnv, amConfig.getTezConfiguration(),
+          TezConfiguration.TEZ_TASK_LAUNCH_ENV,
+          TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT,
+          TezClientUtils.usingTezLibsFromArchive(getTezJarResources(sessionCredentials)));
 
       DAGClientAMProtocolRPC.PreWarmRequestProto.Builder
         preWarmReqProtoBuilder =

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index d99d35d..dda30c2 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -45,9 +45,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -104,6 +102,38 @@ public class TezClientUtils {
   private static Log LOG = LogFactory.getLog(TezClientUtils.class);
   private static final int UTF8_CHUNK_SIZE = 16 * 1024;
 
+  private static FileStatus[] getLRFileStatus(String fileName, Configuration conf,
+                                              boolean asDir) throws
+      IOException {
+
+    URI uri;
+    try {
+      uri = new URI(fileName);
+    } catch (URISyntaxException e) {
+      String message = "Invalid URI defined in configuration for"
+          + " location of TEZ jars. providedURI=" + fileName;
+      LOG.error(message);
+      throw new TezUncheckedException(message, e);
+    }
+
+    if (!uri.isAbsolute()) {
+      String message = "Non-absolute URI defined in configuration for"
+          + " location of TEZ jars. providedURI=" + fileName;
+      LOG.error(message);
+      throw new TezUncheckedException(message);
+    }
+    Path p = new Path(uri);
+    FileSystem pathfs = p.getFileSystem(conf);
+    p = pathfs.makeQualified(p);
+
+    if (asDir) {
+      return pathfs.listStatus(p);
+    } else {
+      FileStatus fStatus = pathfs.getFileStatus(p);
+      return new FileStatus[]{fStatus};
+    }
+  }
+
   /**
    * Setup LocalResource map for Tez jars based on provided Configuration
    * 
@@ -136,48 +166,47 @@ public class TezClientUtils {
      
       List<Path> tezJarPaths = Lists.newArrayListWithCapacity(tezJarUris.length);
 
-      for (String tezJarUri : tezJarUris) {
-        URI uri;
-        try {
-          uri = new URI(tezJarUri.trim());
-        } catch (URISyntaxException e) {
-          String message = "Invalid URI defined in configuration for"
-              + " location of TEZ jars. providedURI=" + tezJarUri;
-          LOG.error(message);
-          throw new TezUncheckedException(message, e);
-        }
-        if (!uri.isAbsolute()) {
-          String message = "Non-absolute URI defined in configuration for"
-              + " location of TEZ jars. providedURI=" + tezJarUri;
-          LOG.error(message);
-          throw new TezUncheckedException(message);
-        }
-        Path p = new Path(uri);
-        FileSystem pathfs = p.getFileSystem(conf);
-        p = pathfs.makeQualified(p);
-        tezJarPaths.add(p);
-        RemoteIterator<LocatedFileStatus> iter = pathfs.listFiles(p, false);
-        while (iter.hasNext()) {
-          LocatedFileStatus fStatus = iter.next();
-          String rsrcName = fStatus.getPath().getName();
-          // FIXME currently not checking for duplicates due to quirks
-          // in assembly generation
-          if (tezJarResources.containsKey(rsrcName)) {
-            String message = "Duplicate resource found"
-                + ", resourceName=" + rsrcName
-                + ", existingPath=" +
-                tezJarResources.get(rsrcName).getResource().toString()
-                + ", newPath=" + fStatus.getPath();
-            LOG.warn(message);
-            // throw new TezUncheckedException(message);
-          }
-          tezJarResources.put(rsrcName,
+      if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz")))
{
+        String fileName = tezJarUris[0];
+        if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
+          FileStatus fStatus = getLRFileStatus(fileName, conf, false)[0];
+          tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME,
               LocalResource.newInstance(
                   ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
-                  LocalResourceType.FILE,
+                  LocalResourceType.ARCHIVE,
                   LocalResourceVisibility.PUBLIC,
                   fStatus.getLen(),
                   fStatus.getModificationTime()));
+          tezJarPaths.add(fStatus.getPath());
+        }
+      } else { // Treat as non-archives - each entry being a directory
+        for (String tezJarUri : tezJarUris) {
+          FileStatus [] fileStatuses = getLRFileStatus(tezJarUri, conf, true);
+          for (FileStatus fStatus : fileStatuses) {
+            if (fStatus.isDirectory()) {
+              // Skip directories - since tez.lib.uris is not recursive.
+              continue;
+            }
+            String rsrcName = fStatus.getPath().getName();
+            // FIXME currently not checking for duplicates due to quirks
+            // in assembly generation
+            if (tezJarResources.containsKey(rsrcName)) {
+              String message = "Duplicate resource found"
+                  + ", resourceName=" + rsrcName
+                  + ", existingPath=" +
+                  tezJarResources.get(rsrcName).getResource().toString()
+                  + ", newPath=" + fStatus.getPath();
+              LOG.warn(message);
+              // throw new TezUncheckedException(message);
+            }
+            tezJarResources.put(rsrcName,
+                LocalResource.newInstance(
+                    ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
+                    LocalResourceType.FILE,
+                    LocalResourceVisibility.PUBLIC,
+                    fStatus.getLen(),
+                    fStatus.getModificationTime()));
+          }
         }
       }
       
@@ -304,6 +333,13 @@ public class TezClientUtils {
     }
   }
 
+  @Private
+  static boolean usingTezLibsFromArchive(Map<String, LocalResource> tezLrs) {
+    return tezLrs.size() == 1 &&
+        tezLrs.keySet().contains(TezConstants.TEZ_TAR_LR_NAME) &&
+        tezLrs.values().iterator().next().getType() == LocalResourceType.ARCHIVE;
+  }
+
   /**
    * Create an ApplicationSubmissionContext to launch a Tez AM
    * @param conf TezConfiguration
@@ -325,6 +361,8 @@ public class TezClientUtils {
 
     Preconditions.checkNotNull(sessionCreds);
 
+    boolean tezLrsAsArchive = usingTezLibsFromArchive(tezJarResources);
+
     FileSystem fs = TezClientUtils.ensureStagingDirExists(conf,
         TezCommonUtils.getTezBaseStagingPath(conf));
     String strAppId = appId.toString();
@@ -415,7 +453,7 @@ public class TezClientUtils {
 
     Map<String, String> environment = new TreeMap<String, String>();
     TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV,
-        TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT);
+        TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive);
     
     // finally apply env set in the code. This could potentially be removed in
     // TEZ-692
@@ -515,7 +553,7 @@ public class TezClientUtils {
         Map<String, String> taskEnv = v.getTaskEnvironment();
         TezYARNUtils.setupDefaultEnv(taskEnv, conf,
             TezConfiguration.TEZ_TASK_LAUNCH_ENV,
-            TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT);
+            TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT, tezLrsAsArchive);
 
         TezClientUtils.setDefaultLaunchCmdOpts(v, amConfig.getTezConfiguration());
       }

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index d2f8c49..72be2ba 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -18,11 +18,14 @@
 package org.apache.tez.common;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.Shell;
@@ -30,13 +33,17 @@ import org.apache.hadoop.util.StringInterner;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
 
 @Private
 public class TezYARNUtils {
   
   private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex());
+
+  private static Log LOG = LogFactory.getLog(TezYARNUtils.class);
   
-  public static String getFrameworkClasspath(Configuration conf) {
+  public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
     Map<String, String> environment = new HashMap<String, String>();
 
     TezYARNUtils.addToEnvironment(environment,
@@ -49,12 +56,25 @@ public class TezYARNUtils {
         Environment.PWD.$() + File.separator + "*",
         File.pathSeparator);
 
-    // Add YARN/COMMON/HDFS jars and conf locations to path
-    for (String c : conf.getStrings(
-        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
-        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+    if (usingArchive) {
+      TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
+          Environment.PWD.$() + File.separator +
+              TezConstants.TEZ_TAR_LR_NAME + File.separator + "*", File.pathSeparator);
+
       TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          c.trim(), File.pathSeparator);
+          Environment.PWD.$() + File.separator +
+              TezConstants.TEZ_TAR_LR_NAME + File.separator + "lib" + File.separator + "*",
+          File.pathSeparator);
+    }
+
+    if (conf.getBoolean(TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS,
+        TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT)) {
+      for (String c : conf.getStrings(
+          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+        TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
+            c.trim(), File.pathSeparator);
+      }
     }
     return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name()));
   }
@@ -133,10 +153,10 @@ public class TezYARNUtils {
   }
 
   public static void setupDefaultEnv(Map<String, String> env,
-      Configuration conf,  String confEnvKey, String confEnvKeyDefault) {
+      Configuration conf,  String confEnvKey, String confEnvKeyDefault, boolean usingArchive)
{
     // Setup the CLASSPATH in environment
     // i.e. add { Hadoop jars, job jar, CWD } to classpath.
-    String classpath = getFrameworkClasspath(conf);
+    String classpath = getFrameworkClasspath(conf, usingArchive);
     TezYARNUtils.addToEnvironment(env,
         ApplicationConstants.Environment.CLASSPATH.name(),
         classpath, File.pathSeparator);

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 5035148..ce46bf3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -412,8 +412,26 @@ public class TezConfiguration extends Configuration {
   public static final String TEZ_CONTAINER_ERR_FILE_NAME = "stderr";
   public static final String TEZ_CONTAINER_OUT_FILE_NAME = "stdout";
 
-
-  /** The location of the Tez libraries which will be localized for DAGs */
+  /**
+   * The location of the Tez libraries which will be localized for DAGs.
+   * This follows the following semantics
+   * <ol>
+   * <li> To use a single .tar.gz or .tgz file (generated by the tez build), the full
path to this
+   * file (including filename) should be specified. The internal structure of the uncompressed
tgz
+   * will be retained under $CWD/tezlib.</li>
+   *
+   * <li> If a single file is specified without the above mentioned extensions - it
will be treated as
+   * a regular file. This means it will not be uncompressed during runtime. </li>
+   *
+   * <li> If multiple entries exist
+   * <ul>
+   * <li> Files: will be treated as regular files (not uncompressed during runtime)
</li>
+   * <li> Directories: all files under the directory (non-recursive) will be made available
(but not
+   * uncompressed during runtime). </li>
+   * <li> All files / contents of directories are flattened into a single directory
- $CWD </li>
+   * </ul>
+   * </ol>
+   */
   public static final String TEZ_LIB_URIS = TEZ_PREFIX + "lib.uris";
   /** 
    * Allows to ignore 'tez.lib.uris'. Useful during development as well as 
@@ -422,6 +440,14 @@ public class TezConfiguration extends Configuration {
    */
   public static final String TEZ_IGNORE_LIB_URIS = TEZ_PREFIX + "ignore.lib.uris";
 
+  /**
+   * Specify whether hadoop libraries required to run Tez should be the ones deployed on
the cluster.
+   * This is disabled by default - with the expectation being that tez.lib.uris has a complete
+   * tez-deployment which contains the hadoop libraries.
+   */
+  public static final String TEZ_USE_CLUSTER_HADOOP_LIBS = TEZ_PREFIX + "use.cluster.hadoop-libs";
+  public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false;
+
   public static final String TEZ_APPLICATION_TYPE = "TEZ";
 
   public static final String TEZ_AM_GROUPING_SPLIT_COUNT = TEZ_AM_PREFIX +

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index 5109a0f..7767762 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -18,9 +18,12 @@
 
 package org.apache.tez.dag.api;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+
 /**
  * Specifies all constant values in Tez
  */
+@InterfaceAudience.Private
 public class TezConstants {
 
   /**
@@ -28,4 +31,6 @@ public class TezConstants {
    */
   public static final String TEZ_SESSION_MODE_CLI_OPTION = "session";
 
+  public static final String TEZ_TAR_LR_NAME = "tezlib";
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml
index e3c0bc5..f051da3 100644
--- a/tez-dist/pom.xml
+++ b/tez-dist/pom.xml
@@ -47,9 +47,7 @@
   </dependencies>
 
   <properties>
-    <package.format>dir</package.format>
-    <!--includeBaseDirectory is not used - replacement does not work in the packaging-->
-    <package.includeBaseDirectory>false</package.includeBaseDirectory>
+    <package.format>tar.gz</package.format>
   </properties>
 
   <profiles>
@@ -66,19 +64,6 @@
         </dependency>
       </dependencies>
     </profile>
-    <profile>
-      <id>dist-tar</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-        <property>
-          <name>tar</name>
-        </property>
-      </activation>
-      <properties>
-        <package.format>tar.gz</package.format>
-        <package.includeBaseDirectory>true</package.includeBaseDirectory>
-      </properties>
-    </profile>
   </profiles>
 
   <build>
@@ -116,11 +101,11 @@
             </goals>
           </execution>
           <execution>
-            <id>package-tez-full</id>
+            <id>package-tez-partial</id>
             <configuration>
-              <finalName>tez-${project.version}-full</finalName>
+              <finalName>tez-${project.version}-partial</finalName>
               <descriptors>
-                <descriptor>src/main/assembly/tez-dist-full.xml</descriptor>
+                <descriptor>src/main/assembly/tez-dist-partial.xml</descriptor>
               </descriptors>
               <formats>
                 <format>${package.format}</format>

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-dist/src/main/assembly/tez-dist-full.xml
----------------------------------------------------------------------
diff --git a/tez-dist/src/main/assembly/tez-dist-full.xml b/tez-dist/src/main/assembly/tez-dist-full.xml
deleted file mode 100644
index b9d9d35..0000000
--- a/tez-dist/src/main/assembly/tez-dist-full.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-  <id>tez-dist</id>
-  <moduleSets>
-    <moduleSet>
-      <useAllReactorProjects>true</useAllReactorProjects>
-      <excludes>
-        <exclude>org.apache.tez:tez-tests</exclude>
-      </excludes>
-      <binaries>
-        <outputDirectory>/</outputDirectory>
-        <unpack>false</unpack>
-        <dependencySets>
-          <dependencySet>
-            <unpack>false</unpack>
-            <scope>runtime</scope>
-            <outputDirectory>/lib</outputDirectory>
-            <excludes>
-              <exclude>*:*:test-jar</exclude>
-              <exclude>org.apache.tez:*</exclude>
-            </excludes>
-          </dependencySet>
-        </dependencySets>
-      </binaries>
-    </moduleSet>
-  </moduleSets>
-</assembly>

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-dist/src/main/assembly/tez-dist-partial.xml
----------------------------------------------------------------------
diff --git a/tez-dist/src/main/assembly/tez-dist-partial.xml b/tez-dist/src/main/assembly/tez-dist-partial.xml
new file mode 100644
index 0000000..8c4a58b
--- /dev/null
+++ b/tez-dist/src/main/assembly/tez-dist-partial.xml
@@ -0,0 +1,51 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+  <id>tez-dist</id>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <moduleSets>
+    <moduleSet>
+      <useAllReactorProjects>true</useAllReactorProjects>
+      <excludes>
+        <exclude>org.apache.tez:tez-tests</exclude>
+      </excludes>
+      <binaries>
+        <outputDirectory>/</outputDirectory>
+        <unpack>false</unpack>
+        <dependencySets>
+          <dependencySet>
+            <unpack>false</unpack>
+            <scope>runtime</scope>
+            <outputDirectory>/lib</outputDirectory>
+            <useTransitiveDependencies>false</useTransitiveDependencies>
+            <excludes>
+              <exclude>*:*:test-jar</exclude>
+              <exclude>org.apache.hadoop:hadoop-common</exclude>
+              <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
+              <exclude>org.apache.hadoop:hadoop-yarn-api</exclude>
+              <exclude>org.apache.hadoop:hadoop-yarn-client</exclude>
+              <exclude>org.apache.hadoop:hadoop-yarn-common</exclude>
+              <exclude>org.apache.tez:*</exclude>
+            </excludes>
+          </dependencySet>
+        </dependencySets>
+      </binaries>
+    </moduleSet>
+  </moduleSets>
+</assembly>

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-dist/src/main/assembly/tez-dist.xml
----------------------------------------------------------------------
diff --git a/tez-dist/src/main/assembly/tez-dist.xml b/tez-dist/src/main/assembly/tez-dist.xml
index dd35242..cb901ef 100644
--- a/tez-dist/src/main/assembly/tez-dist.xml
+++ b/tez-dist/src/main/assembly/tez-dist.xml
@@ -18,6 +18,7 @@
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
   <id>tez-dist</id>
+  <includeBaseDirectory>false</includeBaseDirectory>
   <moduleSets>
     <moduleSet>
       <useAllReactorProjects>true</useAllReactorProjects>
@@ -32,14 +33,8 @@
             <unpack>false</unpack>
             <scope>runtime</scope>
             <outputDirectory>/lib</outputDirectory>
-            <useTransitiveDependencies>false</useTransitiveDependencies>
             <excludes>
               <exclude>*:*:test-jar</exclude>
-              <exclude>org.apache.hadoop:hadoop-common</exclude>
-              <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
-              <exclude>org.apache.hadoop:hadoop-yarn-api</exclude>
-              <exclude>org.apache.hadoop:hadoop-yarn-client</exclude>
-              <exclude>org.apache.hadoop:hadoop-yarn-common</exclude>
               <exclude>org.apache.tez:*</exclude>
             </excludes>
           </dependencySet>

http://git-wip-us.apache.org/repos/asf/tez/blob/cb3338e8/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java b/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java
index 4803e27..43a781c 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java
@@ -76,6 +76,8 @@ public class MiniTezCluster extends MiniYARNCluster {
   @Override
   public void serviceInit(Configuration conf) throws Exception {
     conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
+    // Use libs from cluster since no build is available
+    conf.setBoolean(TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS, true);
     // blacklisting disabled to prevent scheduling issues
     conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
     if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {


Mime
View raw message