tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject git commit: TEZ-553. Mini mrr tez cluster should setup Yarn site automatically (bikas)
Date Tue, 15 Oct 2013 01:30:19 GMT
Updated Branches:
  refs/heads/master f30470bc7 -> ef782312e


TEZ-553. Mini mrr tez cluster should setup Yarn site automatically (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/ef782312
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/ef782312
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/ef782312

Branch: refs/heads/master
Commit: ef782312eb09d94b5373d18a293d19e332d13228
Parents: f30470b
Author: Bikas Saha <bikas@apache.org>
Authored: Mon Oct 14 18:27:33 2013 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Mon Oct 14 18:27:33 2013 -0700

----------------------------------------------------------------------
 .../org/apache/tez/client/TezClientUtils.java   | 65 ++++++++--------
 .../java/org/apache/tez/client/TezSession.java  | 14 ++++
 .../apache/tez/mapreduce/MiniMRRTezCluster.java | 22 ++++++
 .../org/apache/tez/mapreduce/TestMRRJobs.java   | 38 ----------
 .../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 78 +-------------------
 .../apache/tez/mapreduce/hadoop/MRHelpers.java  | 28 -------
 6 files changed, 71 insertions(+), 174 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef782312/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 87042e9..7209435 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -104,9 +104,6 @@ public class TezClientUtils {
       throws IOException {
     Map<String, LocalResource> tezJarResources =
         new TreeMap<String, LocalResource>();
-    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
-      return tezJarResources;
-    }
 
     // Add tez jars to local resource
     String[] tezJarUris = conf.getStrings(
@@ -277,37 +274,14 @@ public class TezClientUtils {
     }
     vargsFinal.add(mergedCommand.toString());
 
-    LOG.debug("Command to launch container for ApplicationMaster is : "
-        + mergedCommand);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Command to launch container for ApplicationMaster is : "
+          + mergedCommand);
+    }
 
     // Setup the CLASSPATH in environment
     // i.e. add { Hadoop jars, job jar, CWD } to classpath.
-    Map<String, String> environment = new HashMap<String, String>();
-
-    boolean isMiniCluster =
-        conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false);
-    if (isMiniCluster) {
-      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          System.getProperty("java.class.path"));
-    }
-
-    Apps.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$());
-
-    Apps.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$() + File.separator + "*");
-
-    // Add YARN/COMMON/HDFS jars to path
-    if (!isMiniCluster) {
-      for (String c : conf.getStrings(
-          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
-          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
-        Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
-            c.trim());
-      }
-    }
+    Map<String, String> environment = createEnvironment(conf);
 
     if (amConfig.getEnv() != null) {
       for (Map.Entry<String, String> entry : amConfig.getEnv().entrySet()) {
@@ -393,6 +367,14 @@ public class TezClientUtils {
         v.getTaskLocalResources().putAll(tezJarResources);
         v.getTaskLocalResources().put(TezConfiguration.TEZ_PB_BINARY_CONF_NAME,
             binaryConfLRsrc);
+        Map<String, String> taskEnv = v.getTaskEnvironment();
+        for (Map.Entry<String, String> entry : environment.entrySet()) {
+          String key = entry.getKey();
+          String value = entry.getValue();
+          if (!taskEnv.containsKey(key)) {
+            taskEnv.put(key, value);
+          }
+        }
       }
 
       // emit protobuf DAG file style
@@ -456,6 +438,27 @@ public class TezClientUtils {
     return appContext;
 
   }
+  
+  static Map<String, String> createEnvironment(Configuration conf) {
+    Map<String, String> environment = new HashMap<String, String>();
+
+    Apps.addToEnvironment(environment,
+        Environment.CLASSPATH.name(),
+        Environment.PWD.$());
+
+    Apps.addToEnvironment(environment,
+        Environment.CLASSPATH.name(),
+        Environment.PWD.$() + File.separator + "*");
+
+    // Add YARN/COMMON/HDFS jars and conf locations to path
+    for (String c : conf.getStrings(
+        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+          c.trim());
+    }
+    return environment;
+  }
 
   @VisibleForTesting
   static void addLog4jSystemProperties(String logLevel,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef782312/tez-api/src/main/java/org/apache/tez/client/TezSession.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
index f10629c..53cdf3d 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezSession.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
@@ -140,6 +140,20 @@ public class TezSession {
         + ", sessionName=" + sessionName
         + ", applicationId=" + applicationId);
 
+    // setup env
+    Map<String, String> environment = TezClientUtils
+        .createEnvironment(sessionConfig.getYarnConfiguration());
+    for (Vertex v : dag.getVertices()) {
+      Map<String, String> taskEnv = v.getTaskEnvironment();
+      for (Map.Entry<String, String> entry : environment.entrySet()) {
+        String key = entry.getKey();
+        String value = entry.getValue();
+        if (!taskEnv.containsKey(key)) {
+          taskEnv.put(key, value);
+        }
+      }
+    }
+    
     DAGPlan dagPlan = dag.createDag(sessionConfig.getTezConfiguration());
     SubmitDAGRequestProto requestProto =
         SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan).build();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef782312/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
index 4c22b6e..278e124 100644
--- a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/MiniMRRTezCluster.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.app.DAGAppMaster;
 import org.apache.tez.mapreduce.hadoop.MRConfig;
@@ -76,6 +77,23 @@ public class MiniMRRTezCluster extends MiniYARNCluster {
       conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
           "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
     }
+    
+    if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) {
+      // nothing defined. set quick delete value
+      conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
+    }
+
+    File appJarFile = new File(MiniMRRTezCluster.APPJAR);
+
+    if (!appJarFile.exists()) {
+      String message = "TezAppJar " + MiniMRRTezCluster.APPJAR
+          + " not found. Exiting.";
+      LOG.info(message);
+      throw new TezUncheckedException(message);
+    } else {
+      conf.set(TezConfiguration.TEZ_LIB_URIS, "file://" + appJarFile.getAbsolutePath());
+      LOG.info("Set TEZ-LIB-URI to: " + conf.get(TezConfiguration.TEZ_LIB_URIS));
+    }
 
     // VMEM monitoring disabled, PMEM monitoring enabled.
     conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
@@ -141,6 +159,10 @@ public class MiniMRRTezCluster extends MiniYARNCluster {
       throw new RuntimeException(e);
     }
     confFilePath = new Path(confFile.getAbsolutePath());
+    conf.setStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+        workDir.getAbsolutePath(), System.getProperty("java.class.path"));
+    LOG.info("Setting yarn-site.xml via YARN-APP-CP at: "
+        + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH));
   }
 
   public Path getConfigFilePath() {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef782312/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
index 274d150..b85ddb6 100644
--- a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.mapreduce.Job;
@@ -53,25 +52,11 @@ public class TestMRRJobs {
   protected static MiniDFSCluster dfsCluster;
 
   private static Configuration conf = new Configuration();
-  private static FileSystem localFs;
   private static FileSystem remoteFs;
-  static {
-    try {
-      localFs = FileSystem.getLocal(conf);
-    } catch (IOException io) {
-      throw new RuntimeException("problem getting local fs", io);
-    }
-  }
 
   private static String TEST_ROOT_DIR = "target"
       + Path.SEPARATOR + TestMRRJobs.class.getName() + "-tmpDir";
 
-  private static Path TEST_ROOT_DIR_PATH =
-      localFs.makeQualified(new Path(TEST_ROOT_DIR));
-  static Path APP_JAR = new Path(TEST_ROOT_DIR_PATH, "MRAppJar.jar");
-  static Path YARN_SITE_XML = new Path(TEST_ROOT_DIR_PATH, "yarn-site.xml");
-  static Path APP_JAR_HDFS = new Path("/tmp/localResources/MRAppJar.jar");
-  static Path YARN_SITE_XML_HDFS = new Path("/tmp/localResources/yarn-site.xml");
   private static final String OUTPUT_ROOT_DIR = "/tmp" + Path.SEPARATOR +
       TestMRRJobs.class.getSimpleName();
 
@@ -83,8 +68,6 @@ public class TestMRRJobs {
       dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
         .format(true).racks(null).build();
       remoteFs = dfsCluster.getFileSystem();
-      APP_JAR_HDFS = remoteFs.makeQualified(APP_JAR_HDFS);
-      YARN_SITE_XML_HDFS = remoteFs.makeQualified(YARN_SITE_XML_HDFS);
     } catch (IOException io) {
       throw new RuntimeException("problem starting mini dfs cluster", io);
     }
@@ -106,17 +89,6 @@ public class TestMRRJobs {
       mrrTezCluster.start();
     }
 
-    LOG.info("APP_JAR: " + APP_JAR);
-    LOG.info("APP_JAR_HDFS: " + APP_JAR_HDFS);
-    LOG.info("YARN_SITE_XML: " + YARN_SITE_XML);
-    LOG.info("YARN_SITE_XML_HDFS: " + YARN_SITE_XML_HDFS);
-
-    localFs.copyFromLocalFile(new Path(MiniMRRTezCluster.APPJAR), APP_JAR);
-    localFs.setPermission(APP_JAR, new FsPermission("700"));
-    localFs.copyFromLocalFile(mrrTezCluster.getConfigFilePath(), YARN_SITE_XML);
-
-    remoteFs.copyFromLocalFile(new Path(MiniMRRTezCluster.APPJAR), APP_JAR_HDFS);
-    remoteFs.copyFromLocalFile(mrrTezCluster.getConfigFilePath(), YARN_SITE_XML_HDFS);
   }
 
   @AfterClass
@@ -150,8 +122,6 @@ public class TestMRRJobs {
     Job job = sleepJob.createJob(1, 1, 1, 1, 1,
         1, 1, 1, 1, 1);
 
-    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
-    job.addFileToClassPath(YARN_SITE_XML);
     job.setJarByClass(MRRSleepJob.class);
     job.setMaxMapAttempts(1); // speed up failures
     job.submit();
@@ -187,8 +157,6 @@ public class TestMRRJobs {
     Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
     FileOutputFormat.setOutputPath(job, outputDir);
     job.setSpeculativeExecution(false);
-    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
-    job.addFileToClassPath(YARN_SITE_XML);
     job.setJarByClass(RandomTextWriterJob.class);
     job.setMaxMapAttempts(1); // speed up failures
     job.submit();
@@ -239,8 +207,6 @@ public class TestMRRJobs {
     Job job = sleepJob.createJob(1, 1, 1, 1, 1,
         1, 1, 1, 1, 1);
 
-    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
-    job.addFileToClassPath(YARN_SITE_XML);
     job.setJarByClass(MRRSleepJob.class);
     job.setMaxMapAttempts(1); // speed up failures
     job.getConfiguration().setBoolean(MRRSleepJob.MAP_FATAL_ERROR, true);
@@ -275,8 +241,6 @@ public class TestMRRJobs {
     Job job = sleepJob.createJob(1, 1, 1, 1, 1,
         1, 1, 1, 1, 1);
 
-    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
-    job.addFileToClassPath(YARN_SITE_XML);
     job.setJarByClass(MRRSleepJob.class);
     job.setMaxMapAttempts(3); // speed up failures
     job.getConfiguration().setBoolean(MRRSleepJob.MAP_THROW_ERROR, true);
@@ -310,8 +274,6 @@ public class TestMRRJobs {
     Job job = sleepJob.createJob(1, 1, 2, 1, 1,
         1, 1, 1, 1, 1);
 
-    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
-    job.addFileToClassPath(YARN_SITE_XML);
     job.setJarByClass(MRRSleepJob.class);
     job.setMaxMapAttempts(1); // speed up failures
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef782312/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 3822b4d..a6dbe26 100644
--- a/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-mapreduce-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -18,7 +18,6 @@
 
 package org.apache.tez.mapreduce;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
@@ -30,12 +29,10 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -46,9 +43,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.client.AMConfiguration;
 import org.apache.tez.client.TezClient;
@@ -100,30 +95,11 @@ public class TestMRRJobsDAGApi {
   protected static MiniDFSCluster dfsCluster;
 
   private static Configuration conf = new Configuration();
-  private static FileSystem localFs;
   private static FileSystem remoteFs;
-  static {
-    try {
-      localFs = FileSystem.getLocal(conf);
-    } catch (IOException io) {
-      throw new RuntimeException("problem getting local fs", io);
-    }
-  }
 
   private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR
       + TestMRRJobsDAGApi.class.getName() + "-tmpDir";
 
-  private static String TEST_ROOT_DIR_HDFS = "/tmp" + Path.SEPARATOR
-      + TestMRRJobsDAGApi.class.getSimpleName();
-
-  private static Path TEST_ROOT_DIR_PATH = localFs.makeQualified(new Path(
-      TEST_ROOT_DIR));
-  static Path APP_JAR = new Path(TEST_ROOT_DIR_PATH, "MRAppJar.jar");
-  static Path YARN_SITE_XML = new Path(TEST_ROOT_DIR_PATH, "yarn-site.xml");
-
-  static Path APP_JAR_HDFS = new Path(TEST_ROOT_DIR_HDFS, "MRAppJar.jar");
-  static Path YARN_SITE_XML_HDFS = new Path(TEST_ROOT_DIR_HDFS, "yarn-site.xml");
-
   @BeforeClass
   public static void setup() throws IOException {
     try {
@@ -131,42 +107,19 @@ public class TestMRRJobsDAGApi {
       dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
           .format(true).racks(null).build();
       remoteFs = dfsCluster.getFileSystem();
-      APP_JAR_HDFS = remoteFs.makeQualified(APP_JAR_HDFS);
-      YARN_SITE_XML_HDFS = remoteFs.makeQualified(YARN_SITE_XML_HDFS);
     } catch (IOException io) {
       throw new RuntimeException("problem starting mini dfs cluster", io);
     }
-
-    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
-      LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
-          + " not found. Not running test.");
-      return;
-    }
-
+    
     if (mrrTezCluster == null) {
       mrrTezCluster = new MiniMRRTezCluster(TestMRRJobsDAGApi.class.getName(),
           1, 1, 1);
       Configuration conf = new Configuration();
       conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
-      conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
-      conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
       mrrTezCluster.init(conf);
       mrrTezCluster.start();
     }
 
-    LOG.info("APP_JAR: " + APP_JAR);
-    LOG.info("APP_JAR_HDFS: " + APP_JAR_HDFS);
-    LOG.info("YARN_SITE_XML: " + YARN_SITE_XML);
-    LOG.info("YARN_SITE_XML_HDFS: " + YARN_SITE_XML_HDFS);
-
-    localFs.copyFromLocalFile(new Path(MiniMRRTezCluster.APPJAR), APP_JAR);
-    localFs.setPermission(APP_JAR, new FsPermission("700"));
-    localFs.copyFromLocalFile(mrrTezCluster.getConfigFilePath(), YARN_SITE_XML);
-
-    remoteFs
-        .copyFromLocalFile(new Path(MiniMRRTezCluster.APPJAR), APP_JAR_HDFS);
-    remoteFs.copyFromLocalFile(mrrTezCluster.getConfigFilePath(),
-        YARN_SITE_XML_HDFS);
   }
 
   @AfterClass
@@ -226,19 +179,9 @@ public class TestMRRJobsDAGApi {
         mrrTezCluster.getConfig());
     tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
         remoteStagingDir.toString());
-    LocalResource appJarLr = createLocalResource(remoteFs,
-        remoteFs.makeQualified(APP_JAR_HDFS), LocalResourceType.FILE,
-        LocalResourceVisibility.APPLICATION);
-    LocalResource yarnSiteLr = createLocalResource(remoteFs,
-        remoteFs.makeQualified(YARN_SITE_XML_HDFS), LocalResourceType.FILE,
-        LocalResourceVisibility.APPLICATION);
-    Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
-    commonLocalResources.put(APP_JAR.getName(), appJarLr);
 
     Map<String, LocalResource> amLocalResources =
         new HashMap<String, LocalResource>();
-    amLocalResources.put("yarn-site.xml", yarnSiteLr);
-    amLocalResources.putAll(commonLocalResources);
 
     AMConfiguration amConfig = new AMConfiguration(
         "default", commonEnv, amLocalResources,
@@ -326,9 +269,6 @@ public class TestMRRJobsDAGApi {
 
   private Map<String, String> createCommonEnv() {
     Map<String, String> commonEnv = new HashMap<String, String>();
-    Apps.addToEnvironment(commonEnv, Environment.CLASSPATH.name(), ".");
-    Apps.addToEnvironment(commonEnv, Environment.CLASSPATH.name(),
-        System.getProperty("java.class.path"));
     return commonEnv;
   }
 
@@ -342,12 +282,6 @@ public class TestMRRJobsDAGApi {
       YarnException {
     LOG.info("\n\n\nStarting testMRRSleepJobDagSubmit().");
 
-    if (!(new File(MiniMRRTezCluster.APPJAR)).exists()) {
-      LOG.info("MRAppJar " + MiniMRRTezCluster.APPJAR
-          + " not found. Not running test.");
-      return State.ERROR;
-    }
-
     JobConf stage1Conf = new JobConf(mrrTezCluster.getConfig());
     JobConf stage2Conf = new JobConf(mrrTezCluster.getConfig());
     JobConf stage3Conf = new JobConf(mrrTezCluster.getConfig());
@@ -434,16 +368,7 @@ public class TestMRRJobsDAGApi {
         1, Resource.newInstance(256, 1));
     MRHelpers.addMROutput(stage3Vertex, stage3Payload);
 
-    LocalResource appJarLr = createLocalResource(remoteFs,
-        remoteFs.makeQualified(APP_JAR_HDFS), LocalResourceType.FILE,
-        LocalResourceVisibility.APPLICATION);
-    LocalResource yarnSiteLr = createLocalResource(remoteFs,
-        remoteFs.makeQualified(YARN_SITE_XML_HDFS), LocalResourceType.FILE,
-        LocalResourceVisibility.APPLICATION);
-
     Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
-    commonLocalResources.put(APP_JAR.getName(), appJarLr);
-
     Map<String, String> commonEnv = createCommonEnv();
 
     if (!genSplitsInAM) {
@@ -498,7 +423,6 @@ public class TestMRRJobsDAGApi {
 
     Map<String, LocalResource> amLocalResources =
         new HashMap<String, LocalResource>();
-    amLocalResources.put("yarn-site.xml", yarnSiteLr);
     amLocalResources.putAll(commonLocalResources);
 
     TezConfiguration tezConf = new TezConfiguration(

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/ef782312/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 8e025e9..6736467 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -742,34 +742,6 @@ public class MRHelpers {
    */
   public static void updateEnvironmentForMRTasks(Configuration conf,
       Map<String, String> environment, boolean isMap) {
-
-    boolean isMiniCluster =
-        conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false);
-    if (isMiniCluster) {
-      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          System.getProperty("java.class.path"));
-    }
-
-    // TEZ jars and deps will be localized by the TezClient submission layer
-    // Assumption is that MR is also in TEZ dependencies
-    Apps.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$());
-
-    // Add YARN/COMMON/HDFS jars to path
-    if (!isMiniCluster) {
-      for (String c : conf.getStrings(
-          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
-          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
-        Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
-            c.trim());
-      }
-    }
-
-    Apps.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$() + File.separator + "*");
-
     // Shell
     environment.put(Environment.SHELL.name(), conf.get(
         MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));


Mime
View raw message