tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject git commit: TEZ-1127. Add TEZ_TASK_JAVA_OPTS and TEZ_ENV configs to specify values from config
Date Thu, 26 Jun 2014 02:16:23 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master 9ddc6e045 -> af538639c


TEZ-1127. Add TEZ_TASK_JAVA_OPTS and TEZ_ENV configs to specify values from config


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/af538639
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/af538639
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/af538639

Branch: refs/heads/master
Commit: af538639c88ec7e5d7a19b73908ec64fcc88b95e
Parents: 9ddc6e0
Author: Bikas Saha <bikas@apache.org>
Authored: Wed Jun 25 19:16:12 2014 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Wed Jun 25 19:16:12 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../org/apache/tez/client/AMConfiguration.java  |  3 -
 .../org/apache/tez/client/TezClientUtils.java   | 90 ++++++++++++--------
 .../java/org/apache/tez/client/TezSession.java  | 11 ++-
 .../org/apache/tez/common/TezYARNUtils.java     | 87 ++++++++++++++++++-
 .../apache/tez/dag/api/TezConfiguration.java    | 45 ++++++++--
 .../java/org/apache/tez/dag/api/Vertex.java     |  8 +-
 .../examples/BroadcastAndOneToOneExample.java   |  8 +-
 .../mapreduce/examples/FilterLinesByWord.java   | 11 +--
 .../examples/FilterLinesByWordOneToOne.java     | 11 +--
 .../mapreduce/examples/IntersectDataGen.java    |  3 +-
 .../mapreduce/examples/IntersectExample.java    | 26 +++---
 .../mapreduce/examples/IntersectValidate.java   | 18 ++--
 .../tez/mapreduce/examples/MRRSleepJob.java     | 16 ----
 .../mapreduce/examples/OrderedWordCount.java    | 16 ----
 .../tez/mapreduce/examples/UnionExample.java    |  8 --
 .../tez/mapreduce/examples/WordCount.java       |  6 --
 .../apache/tez/mapreduce/client/YARNRunner.java |  2 +-
 .../apache/tez/mapreduce/hadoop/MRHelpers.java  |  8 +-
 .../apache/tez/mapreduce/TestMRRJobsDAGApi.java |  8 --
 .../org/apache/tez/test/TestDAGRecovery.java    |  2 +-
 .../org/apache/tez/test/TestDAGRecovery2.java   |  2 +-
 22 files changed, 216 insertions(+), 175 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2086dca..abfcd27 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -13,6 +13,8 @@ INCOMPATIBLE CHANGES
   scheduled tasks
   TEZ-1169. Allow numPhysicalInputs to be specified for RootInputs.
   TEZ-1131. Simplify EdgeManager APIs
+  TEZ-1127. Add TEZ_TASK_JAVA_OPTS and TEZ_ENV configs to specify values from
+  config
 
 Release 0.4.0-incubating: 2014-04-05
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
index 9891daa..7c9dd3c 100644
--- a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
@@ -63,9 +63,6 @@ public class AMConfiguration {
     this.queueName = this.amConf.get(TezConfiguration.TEZ_QUEUE_NAME);
 
     this.env = new HashMap<String, String>();
-    TezYARNUtils.setEnvFromInputString(this.env,
-        this.amConf.get(TezConfiguration.TEZ_AM_ENV),
-        File.pathSeparator);
     if (env != null) {
       this.env.putAll(env);
     }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 6cebde9..bae0328 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -375,13 +375,18 @@ public class TezClientUtils {
     List<String> vargs = new ArrayList<String>(8);
     vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
 
+    String amOpts = amConfig.getAMConf().get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
+        TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);
+    if (amOpts != null && !amOpts.isEmpty()) {
+      vargs.add(amOpts);
+    }
+
     String amLogLevel = amConfig.getAMConf().get(
         TezConfiguration.TEZ_AM_LOG_LEVEL,
         TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
-    addLog4jSystemProperties(amLogLevel, vargs);
 
-    vargs.add(amConfig.getAMConf().get(TezConfiguration.TEZ_AM_JAVA_OPTS,
-        TezConfiguration.TEZ_AM_JAVA_OPTS_DEFAULT));
+    maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs);
+
     // FIX sun bug mentioned in TEZ-327
     vargs.add("-Dsun.nio.ch.bugLevel=''");
 
@@ -409,13 +414,11 @@ public class TezClientUtils {
           + mergedCommand);
     }
 
-    // Setup the CLASSPATH in environment
-    // i.e. add { Hadoop jars, job jar, CWD } to classpath.
-    String classpath = getFrameworkClasspath(conf);
-
     Map<String, String> environment = new TreeMap<String, String>();
-    environment.put(Environment.CLASSPATH.name(), classpath);
-
+    TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV,
+        TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT);
+    
+    // finally apply env set in the code. This could potentially be removed in TEZ-692
     if (amConfig.getEnv() != null) {
       for (Map.Entry<String, String> entry : amConfig.getEnv().entrySet()) {
         TezYARNUtils.addToEnvironment(environment, entry.getKey(), entry.getValue(),
@@ -510,13 +513,11 @@ public class TezClientUtils {
             binaryConfLRsrc);
 
         Map<String, String> taskEnv = v.getTaskEnvironment();
-        for (Map.Entry<String, String> entry : environment.entrySet()) {
-          String key = entry.getKey();
-          String value = entry.getValue();
-          if (!taskEnv.containsKey(key)) {
-            taskEnv.put(key, value);
-          }
-        }
+        TezYARNUtils.setupDefaultEnv(taskEnv, conf,
+            TezConfiguration.TEZ_TASK_LAUNCH_ENV,
+            TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT);
+
+        TezClientUtils.setDefaultLaunchCmdOpts(v, amConfig.getAMConf());
       }
 
       // emit protobuf DAG file style
@@ -588,27 +589,44 @@ public class TezClientUtils {
 
   }
   
-  static String getFrameworkClasspath(Configuration conf) {
-    Map<String, String> environment = new HashMap<String, String>();
-
-    TezYARNUtils.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$(),
-        File.pathSeparator);
-
-    TezYARNUtils.addToEnvironment(environment,
-        Environment.CLASSPATH.name(),
-        Environment.PWD.$() + File.separator + "*",
-        File.pathSeparator);
-
-    // Add YARN/COMMON/HDFS jars and conf locations to path
-    for (String c : conf.getStrings(
-        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
-        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
-      TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          c.trim(), File.pathSeparator);
+  
+  static void maybeAddDefaultLoggingJavaOpts(String logLevel, List<String> vargs) {
+    if (vargs != null && !vargs.isEmpty()) {
+      for (String arg : vargs) {
+        if (arg.contains(TezConfiguration.TEZ_ROOT_LOGGER_NAME)) {
+          return ;
+        }
+      }
+    }
+    TezClientUtils.addLog4jSystemProperties(logLevel, vargs);
+  }
+  
+  static String maybeAddDefaultLoggingJavaOpts(String logLevel, String javaOpts) {
+    List<String> vargs = new ArrayList<String>(5);
+    if (javaOpts != null) {
+      vargs.add(javaOpts);
+    } else {
+      vargs.add("");
     }
-    return environment.get(Environment.CLASSPATH.name());
+    maybeAddDefaultLoggingJavaOpts(logLevel, vargs);
+    if (vargs.size() == 1) {
+      return vargs.get(0);
+    }
+    return StringUtils.join(vargs, " ").trim();
+  }
+  
+  static void setDefaultLaunchCmdOpts(Vertex v, TezConfiguration conf) {
+    String vOpts = v.getJavaOpts();
+    String vConfigOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS,
+        TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT);
+    if (vConfigOpts != null && vConfigOpts.length() > 0) {
+      vOpts += (" " + vConfigOpts);
+    }
+    
+    vOpts = maybeAddDefaultLoggingJavaOpts(conf.get(
+        TezConfiguration.TEZ_TASK_LOG_LEVEL,
+        TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT), vOpts);
+    v.setJavaOpts(vOpts);
   }
 
   @Private

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-api/src/main/java/org/apache/tez/client/TezSession.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
index a085774..d73e9eb 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezSession.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
@@ -211,14 +211,13 @@ public class TezSession {
     // Obtain DAG specific credentials.
     TezClientUtils.setupDAGCredentials(dag, sessionCredentials, sessionConfig.getTezConfiguration());
 
+    // TODO TEZ-1229 - fix jar resources
     // setup env
-    String classpath = TezClientUtils
-        .getFrameworkClasspath(sessionConfig.getYarnConfiguration());
     for (Vertex v : dag.getVertices()) {
       Map<String, String> taskEnv = v.getTaskEnvironment();
-      TezYARNUtils.addToEnvironment(taskEnv,
-          ApplicationConstants.Environment.CLASSPATH.name(),
-          classpath, File.pathSeparator);
+      TezYARNUtils.setupDefaultEnv(taskEnv, sessionConfig.getTezConfiguration(),
+          TezConfiguration.TEZ_TASK_LAUNCH_ENV, TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT);
+      TezClientUtils.setDefaultLaunchCmdOpts(v, sessionConfig.getTezConfiguration());
     }
     
     DAGPlan dagPlan = dag.createDag(sessionConfig.getTezConfiguration());
@@ -362,7 +361,7 @@ public class TezSession {
             + " timeout interval, timeoutSecs=" + clientTimeout);
       }
 
-      String classpath = TezClientUtils
+      String classpath = TezYARNUtils
         .getFrameworkClasspath(sessionConfig.getYarnConfiguration());
       Map<String, String> contextEnv = context.getEnvironment();
       TezYARNUtils.addToEnvironment(contextEnv,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index bcd0393..d2f8c49 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -17,21 +17,50 @@
 
 package org.apache.tez.common;
 
+import java.io.File;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringInterner;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
 @Private
 public class TezYARNUtils {
   
   private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex());
+  
+  public static String getFrameworkClasspath(Configuration conf) {
+    Map<String, String> environment = new HashMap<String, String>();
+
+    TezYARNUtils.addToEnvironment(environment,
+        Environment.CLASSPATH.name(),
+        Environment.PWD.$(),
+        File.pathSeparator);
+
+    TezYARNUtils.addToEnvironment(environment,
+        Environment.CLASSPATH.name(),
+        Environment.PWD.$() + File.separator + "*",
+        File.pathSeparator);
 
-  public static void setEnvFromInputString(Map<String, String> env,
-      String envString,  String classPathSeparator) {
+    // Add YARN/COMMON/HDFS jars and conf locations to path
+    for (String c : conf.getStrings(
+        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+      TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
+          c.trim(), File.pathSeparator);
+    }
+    return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name()));
+  }
+
+  public static void appendToEnvFromInputString(Map<String, String> env,
+      String envString, String classPathSeparator) {
     if (envString != null && envString.length() > 0) {
       String childEnvs[] = envString.split(",");
       for (String cEnv : childEnvs) {
@@ -46,7 +75,7 @@ public class TezYARNUtils {
           // from the tt's env
           if (replace == null)
             replace = System.getenv(var);
-          // If the env key is note present leave it as it is and assume it will
+          // If the env key is not present leave it as it is and assume it will
           // be set by YARN ContainerLauncher. For eg: $HADOOP_COMMON_HOME
           if (replace != null)
             m.appendReplacement(sb, Matcher.quoteReplacement(replace));
@@ -57,6 +86,33 @@ public class TezYARNUtils {
     }
   }
   
+  public static void setEnvIfAbsentFromInputString(Map<String, String> env,
+      String envString) {
+    if (envString != null && envString.length() > 0) {
+      String childEnvs[] = envString.split(",");
+      for (String cEnv : childEnvs) {
+        String[] parts = cEnv.split("="); // split on '='
+        Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
+        StringBuffer sb = new StringBuffer();
+        while (m.find()) {
+          String var = m.group(1);
+          // replace $env with the child's env constructed by tt's
+          String replace = env.get(var);
+          // if this key is not configured by the tt for the child .. get it
+          // from the tt's env
+          if (replace == null)
+            replace = System.getenv(var);
+          // If the env key is not present leave it as it is and assume it will
+          // be set by YARN ContainerLauncher. For eg: $HADOOP_COMMON_HOME
+          if (replace != null)
+            m.appendReplacement(sb, Matcher.quoteReplacement(replace));
+        }
+        m.appendTail(sb);
+        putIfAbsent(env, parts[0], sb.toString());
+      }
+    }
+  }
+  
   public static void addToEnvironment(
       Map<String, String> environment,
       String variable, String value, String classPathSeparator) {
@@ -70,4 +126,29 @@ public class TezYARNUtils {
         StringInterner.weakIntern(val));
   }
 
+  private static void putIfAbsent(Map<String, String> env, String key, String value) {
+    if (!env.containsKey(key)) {
+      env.put(StringInterner.weakIntern(key), StringInterner.weakIntern(value));
+    }
+  }
+
+  public static void setupDefaultEnv(Map<String, String> env,
+      Configuration conf,  String confEnvKey, String confEnvKeyDefault) {
+    // Setup the CLASSPATH in environment
+    // i.e. add { Hadoop jars, job jar, CWD } to classpath.
+    String classpath = getFrameworkClasspath(conf);
+    TezYARNUtils.addToEnvironment(env,
+        ApplicationConstants.Environment.CLASSPATH.name(),
+        classpath, File.pathSeparator);
+
+    // set any env from config if it is not set already
+    TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get(
+        confEnvKey, confEnvKeyDefault));
+    
+    // Append pwd to LD_LIBRARY_PATH
+    // Done separately here because this is known to work platform independent
+    TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(),
+        Environment.PWD.$(), File.pathSeparator);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 97a35eb..8c1bf8a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -51,24 +51,51 @@ public class TezConfiguration extends Configuration {
       "org.apache.tez.dag.app.DAGAppMaster";
 
   /** Root Logging level passed to the Tez app master.*/
-  public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX+"log.level";
+  public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX + "log.level";
   public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO";
 
+  /** Root Logging level passed to the Tez app master.*/
+  public static final String TEZ_TASK_LOG_LEVEL = TEZ_TASK_PREFIX + "log.level";
+  public static final String TEZ_TASK_LOG_LEVEL_DEFAULT = "INFO";
+
   public static final String TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS =
       TEZ_AM_PREFIX + "commit-all-outputs-on-dag-success";
   public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true;
 
-  /** Java options for the Tez AppMaster process. */
-  public static final String TEZ_AM_JAVA_OPTS = TEZ_AM_PREFIX
-      + "java.opts";
-  public static final String TEZ_AM_JAVA_OPTS_DEFAULT = " -Xmx1024m ";
-
-  /** User-provided env for the Tez AM. Any env provided in AMConfiguration
-   * overrides env defined by this config property
+  /** Command line options for the Tez AppMaster process. */
+  public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX +  "java.opts";
+  public static final String TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT = 
+      "-Djava.net.preferIPv4Stack=true " +
+      "-Dhadoop.metrics.log.level=WARN " + 
+      "-Xmx1024m"; // Remove after TEZ-699
+
+  /** Command line options for the Tez Task processes. */
+  public static final String TEZ_TASK_LAUNCH_CMD_OPTS = TEZ_TASK_PREFIX
+      + "launch.cmd-opts";
+  public static final String TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT = 
+      "-Djava.net.preferIPv4Stack=true " +
+      "-Dhadoop.metrics.log.level=WARN " + 
+      "-Xmx200m"; // Remove after TEZ-699
+
+  /** Env settings for the Tez AppMaster process.
+   * Should be specified as a comma-separated of key-value pairs where each pair
+   * is defined as KEY=VAL
+   * e.g. "LD_LIBRARY_PATH=.,USERNAME=foo"
+   * These take least precedence compared to other methods of setting env
+  */
+  public static final String TEZ_AM_LAUNCH_ENV = TEZ_AM_PREFIX
+      + "launch.env";
+  public static final String TEZ_AM_LAUNCH_ENV_DEFAULT = "";
+  
+  /** Env settings for the Tez Task processes.
    * Should be specified as a comma-separated of key-value pairs where each pair
    * is defined as KEY=VAL
+   * e.g. "LD_LIBRARY_PATH=.,USERNAME=foo"
+   * These take least precedence compared to other methods of setting env
    */
-  public static final String TEZ_AM_ENV = TEZ_AM_PREFIX + "env";
+  public static final String TEZ_TASK_LAUNCH_ENV = TEZ_TASK_PREFIX
+      + "launch.env";
+  public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = "";
 
   public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN = TEZ_AM_PREFIX +
       "am.complete.cancel.delegation.tokens";

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index a6c5999..007884a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -127,12 +127,8 @@ public class Vertex {
   }
 
   public Vertex setTaskEnvironment(Map<String, String> environment) {
-    if (environment == null) {
-      this.taskEnvironment = new HashMap<String, String>();
-    }
-    else {
-      this.taskEnvironment = environment;
-    }
+    Preconditions.checkArgument(environment != null);
+    this.taskEnvironment.putAll(environment);
     return this;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index 3bb4223..d028775 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -172,20 +172,17 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
     Vertex broadcastVertex = new Vertex("Broadcast", new ProcessorDescriptor(
         InputProcessor.class.getName()),
         numBroadcastTasks, MRHelpers.getMapResource(kvInputConf));
-    broadcastVertex.setJavaOpts(MRHelpers.getMapJavaOpts(kvInputConf));
     
     Vertex inputVertex = new Vertex("Input", new ProcessorDescriptor(
         InputProcessor.class.getName()).setUserPayload(procPayload),
         numOneToOneTasks, MRHelpers.getMapResource(kvInputConf));
-    inputVertex.setJavaOpts(MRHelpers.getMapJavaOpts(kvInputConf));
     
     byte[] kvOneToOnePayload = MRHelpers.createUserPayloadFromConf(kvOneToOneConf);
     Vertex oneToOneVertex = new Vertex("OneToOne",
         new ProcessorDescriptor(
             OneToOneProcessor.class.getName()).setUserPayload(procPayload),
             -1, MRHelpers.getReduceResource(kvOneToOneConf));
-    oneToOneVertex.setJavaOpts(
-        MRHelpers.getReduceJavaOpts(kvOneToOneConf)).setVertexManagerPlugin(
+    oneToOneVertex.setVertexManagerPlugin(
             new VertexManagerPluginDescriptor(InputReadyVertexManager.class.getName()));
     
     DAG dag = new DAG("BroadcastAndOneToOneExample");
@@ -242,9 +239,6 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
     TokenCache.obtainTokensForNamenodes(credentials, new Path[] {stagingDir}, tezConf);
     TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
 
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
-        MRHelpers.getMRAMJavaOpts(tezConf));
-
     // No need to add jar containing this class as assumed to be part of
     // the tez jars.
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index dac35a7..423bb16 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -146,8 +146,6 @@ public class FilterLinesByWord extends Configured implements Tool {
     tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
     TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
 
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, MRHelpers.getMRAMJavaOpts(tezConf));
-
     String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class);
     if (jarPath == null) {
       throw new TezUncheckedException("Could not find any jar containing"
@@ -207,7 +205,6 @@ public class FilterLinesByWord extends Configured implements Tool {
     Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
         FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload),
         stage1NumTasks, MRHelpers.getMapResource(stage1Conf));
-    stage1Vertex.setJavaOpts(MRHelpers.getMapJavaOpts(stage1Conf));
     if (generateSplitsInClient) {
       stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
       Map<String, LocalResource> stage1LocalResources = new HashMap<String, LocalResource>();
@@ -217,9 +214,6 @@ public class FilterLinesByWord extends Configured implements Tool {
     } else {
       stage1Vertex.setTaskLocalResources(commonLocalResources);
     }
-    Map<String, String> stage1Env = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(stage1Conf, stage1Env, true);
-    stage1Vertex.setTaskEnvironment(stage1Env);
 
     // Configure the Input for stage1
     Class<? extends TezRootInputInitializer> initializerClazz = generateSplitsInClient ? null
@@ -234,10 +228,7 @@ public class FilterLinesByWord extends Configured implements Tool {
         FilterByWordOutputProcessor.class.getName()).setUserPayload(MRHelpers
         .createUserPayloadFromConf(stage2Conf)), 1,
         MRHelpers.getReduceResource(stage2Conf));
-    stage2Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage2Conf)).setTaskLocalResources(commonLocalResources);
-    Map<String, String> stage2Env = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(stage2Conf, stage2Env, false);
-    stage2Vertex.setTaskEnvironment(stage2Env);
+    stage2Vertex.setTaskLocalResources(commonLocalResources);
 
     // Configure the Output for stage2
     OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index bf765cb..1518cf1 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -135,8 +135,6 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
     tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
     TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
 
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, MRHelpers.getMRAMJavaOpts(tezConf));
-
     String jarPath = ClassUtil.findContainingJar(FilterLinesByWordOneToOne.class);
     if (jarPath == null) {
       throw new TezUncheckedException("Could not find any jar containing"
@@ -192,7 +190,6 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
     Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
         FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload),
         stage1NumTasks, MRHelpers.getMapResource(stage1Conf));
-    stage1Vertex.setJavaOpts(MRHelpers.getMapJavaOpts(stage1Conf));
     if (generateSplitsInClient) {
       stage1Vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
       Map<String, LocalResource> stage1LocalResources = new HashMap<String, LocalResource>();
@@ -202,9 +199,6 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
     } else {
       stage1Vertex.setTaskLocalResources(commonLocalResources);
     }
-    Map<String, String> stage1Env = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(stage1Conf, stage1Env, true);
-    stage1Vertex.setTaskEnvironment(stage1Env);
 
     // Configure the Input for stage1
     Class<? extends TezRootInputInitializer> initializerClazz = generateSplitsInClient ? null
@@ -219,10 +213,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
         FilterByWordOutputProcessor.class.getName()).setUserPayload(MRHelpers
         .createUserPayloadFromConf(stage2Conf)), stage1NumTasks,
         MRHelpers.getMapResource(stage2Conf));
-    stage2Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage2Conf)).setTaskLocalResources(commonLocalResources);
-    Map<String, String> stage2Env = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(stage2Conf, stage2Env, false);
-    stage2Vertex.setTaskEnvironment(stage2Env);
+    stage2Vertex.setTaskLocalResources(commonLocalResources);
 
     // Configure the Output for stage2
     stage2Vertex.addOutput("MROutput",

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
index d977701..00611d7 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
@@ -216,8 +216,7 @@ public class IntersectDataGen extends Configured implements Tool {
 
     Vertex genDataVertex = new Vertex("datagen", new ProcessorDescriptor(
         GenDataProcessor.class.getName()).setUserPayload(GenDataProcessor.createConfiguration(
-        largeOutSizePerTask, smallOutSizePerTask)), numTasks, MRHelpers.getMapResource(tezConf))
-        .setJavaOpts(MRHelpers.getMapJavaOpts(tezConf));
+        largeOutSizePerTask, smallOutSizePerTask)), numTasks, MRHelpers.getMapResource(tezConf));
     genDataVertex.addOutput(STREAM_OUTPUT_NAME,
         new OutputDescriptor(MROutput.class.getName()).setUserPayload(streamOutputPayload),
         MROutputCommitter.class);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
index a412df4..68e867c 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
@@ -230,23 +230,23 @@ public class IntersectExample extends Configured implements Tool {
         TextOutputFormat.class.getName(), true);
 
     // Change the way resources are setup - no MRHelpers
-    Vertex streamFileVertex = new Vertex("partitioner1", new ProcessorDescriptor(
-        ForwardingProcessor.class.getName()), -1, MRHelpers.getMapResource(tezConf)).setJavaOpts(
-        MRHelpers.getMapJavaOpts(tezConf)).addInput("streamfile",
-        new InputDescriptor(MRInput.class.getName()).setUserPayload(streamInputPayload),
-        MRInputAMSplitGenerator.class);
+    Vertex streamFileVertex = new Vertex("partitioner1",
+        new ProcessorDescriptor(ForwardingProcessor.class.getName()), -1,
+        MRHelpers.getMapResource(tezConf)).addInput("streamfile",
+        new InputDescriptor(MRInput.class.getName())
+            .setUserPayload(streamInputPayload), MRInputAMSplitGenerator.class);
 
     Vertex hashFileVertex = new Vertex("partitioner2", new ProcessorDescriptor(
-        ForwardingProcessor.class.getName()), -1, MRHelpers.getMapResource(tezConf)).setJavaOpts(
-        MRHelpers.getMapJavaOpts(tezConf)).addInput("hashfile",
-        new InputDescriptor(MRInput.class.getName()).setUserPayload(hashInputPayload),
-        MRInputAMSplitGenerator.class);
+        ForwardingProcessor.class.getName()), -1,
+        MRHelpers.getMapResource(tezConf)).addInput("hashfile",
+        new InputDescriptor(MRInput.class.getName())
+            .setUserPayload(hashInputPayload), MRInputAMSplitGenerator.class);
 
     Vertex intersectVertex = new Vertex("intersect", new ProcessorDescriptor(
-        IntersectProcessor.class.getName()), numPartitions, MRHelpers.getReduceResource(tezConf))
-        .setJavaOpts(MRHelpers.getReduceJavaOpts(tezConf)).addOutput("finalOutput",
-            new OutputDescriptor(MROutput.class.getName()).setUserPayload(finalOutputPayload),
-            MROutputCommitter.class);
+        IntersectProcessor.class.getName()), numPartitions,
+        MRHelpers.getReduceResource(tezConf)).addOutput("finalOutput",
+        new OutputDescriptor(MROutput.class.getName())
+            .setUserPayload(finalOutputPayload), MROutputCommitter.class);
 
     Edge e1 = new Edge(streamFileVertex, intersectVertex, new EdgeProperty(
         DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
index 537eb91..dc82f80 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
@@ -235,20 +235,20 @@ public class IntersectValidate extends Configured implements Tool {
 
     // Change the way resources are setup - no MRHelpers
     Vertex lhsVertex = new Vertex(LHS_INPUT_NAME, new ProcessorDescriptor(
-        ForwardingProcessor.class.getName()), -1, MRHelpers.getMapResource(tezConf)).setJavaOpts(
-        MRHelpers.getMapJavaOpts(tezConf)).addInput("lhs",
-        new InputDescriptor(MRInput.class.getName()).setUserPayload(streamInputPayload),
+        ForwardingProcessor.class.getName()), -1,
+        MRHelpers.getMapResource(tezConf)).addInput("lhs", new InputDescriptor(
+        MRInput.class.getName()).setUserPayload(streamInputPayload),
         MRInputAMSplitGenerator.class);
 
     Vertex rhsVertex = new Vertex(RHS_INPUT_NAME, new ProcessorDescriptor(
-        ForwardingProcessor.class.getName()), -1, MRHelpers.getMapResource(tezConf)).setJavaOpts(
-        MRHelpers.getMapJavaOpts(tezConf)).addInput("rhs",
-        new InputDescriptor(MRInput.class.getName()).setUserPayload(hashInputPayload),
+        ForwardingProcessor.class.getName()), -1,
+        MRHelpers.getMapResource(tezConf)).addInput("rhs", new InputDescriptor(
+        MRInput.class.getName()).setUserPayload(hashInputPayload),
         MRInputAMSplitGenerator.class);
 
-    Vertex intersectValidateVertex = new Vertex("intersectvalidate", new ProcessorDescriptor(
-        IntersectValidateProcessor.class.getName()), numPartitions,
-        MRHelpers.getReduceResource(tezConf)).setJavaOpts(MRHelpers.getReduceJavaOpts(tezConf));
+    Vertex intersectValidateVertex = new Vertex("intersectvalidate",
+        new ProcessorDescriptor(IntersectValidateProcessor.class.getName()),
+        numPartitions, MRHelpers.getReduceResource(tezConf));
 
     Edge e1 = new Edge(lhsVertex, intersectValidateVertex, new EdgeProperty(
         DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index 399a082..2f443ca 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -590,7 +590,6 @@ public class MRRSleepJob extends Configured implements Tool {
     Vertex mapVertex = new Vertex("map", new ProcessorDescriptor(
         MapProcessor.class.getName()).setUserPayload(mapUserPayload),
         numTasks, MRHelpers.getMapResource(mapStageConf));
-    mapVertex.setJavaOpts(MRHelpers.getMapJavaOpts(mapStageConf));
     if (!generateSplitsInAM) {
       mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
     }
@@ -605,9 +604,6 @@ public class MRRSleepJob extends Configured implements Tool {
       mapVertex.setTaskLocalResources(commonLocalResources);
     }
 
-    Map<String, String> mapEnv = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(mapStageConf, mapEnv, true);
-    mapVertex.setTaskEnvironment(mapEnv);
     if (generateSplitsInAM) {
       MRHelpers.addMRInput(mapVertex, mapInputPayload, MRInputAMSplitGenerator.class);
     } else {
@@ -629,11 +625,7 @@ public class MRRSleepJob extends Configured implements Tool {
                 new ProcessorDescriptor(ReduceProcessor.class.getName()).
                 setUserPayload(iReduceUserPayload), numIReducer,
                 MRHelpers.getReduceResource(iconf));
-        ivertex.setJavaOpts(MRHelpers.getReduceJavaOpts(iconf));
         ivertex.setTaskLocalResources(commonLocalResources);
-        Map<String, String> reduceEnv = new HashMap<String, String>();
-        MRHelpers.updateEnvironmentForMRTasks(iconf, reduceEnv, false);
-        ivertex.setTaskEnvironment(reduceEnv);
         vertices.add(ivertex);
       }
     }
@@ -644,12 +636,7 @@ public class MRRSleepJob extends Configured implements Tool {
       finalReduceVertex = new Vertex("reduce", new ProcessorDescriptor(
           ReduceProcessor.class.getName()).setUserPayload(reducePayload),
           numReducer, MRHelpers.getReduceResource(finalReduceConf));
-      finalReduceVertex.setJavaOpts(
-          MRHelpers.getReduceJavaOpts(finalReduceConf));
       finalReduceVertex.setTaskLocalResources(commonLocalResources);
-      Map<String, String> reduceEnv = new HashMap<String, String>();
-      MRHelpers.updateEnvironmentForMRTasks(finalReduceConf, reduceEnv, false);
-      finalReduceVertex.setTaskEnvironment(reduceEnv);
       MRHelpers.addMROutputLegacy(finalReduceVertex, reducePayload);
       vertices.add(finalReduceVertex);
     } else {
@@ -833,9 +820,6 @@ public class MRRSleepJob extends Configured implements Tool {
         mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
         iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);
 
-    conf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
-        MRHelpers.getMRAMJavaOpts(conf));
-
     AMConfiguration amConfig = new AMConfiguration(null,
         null, conf, this.credentials);
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index e8139ac..2acf1b8 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -228,7 +228,6 @@ public class OrderedWordCount extends Configured implements Tool {
     Vertex mapVertex = new Vertex("initialmap", new ProcessorDescriptor(
         MapProcessor.class.getName()).setUserPayload(mapPayload),
         numMaps, MRHelpers.getMapResource(mapStageConf));
-    mapVertex.setJavaOpts(MRHelpers.getMapJavaOpts(mapStageConf));
     if (generateSplitsInClient) {
       mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
       Map<String, LocalResource> mapLocalResources =
@@ -241,9 +240,6 @@ public class OrderedWordCount extends Configured implements Tool {
       mapVertex.setTaskLocalResources(commonLocalResources);
     }
 
-    Map<String, String> mapEnv = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(mapStageConf, mapEnv, true);
-    mapVertex.setTaskEnvironment(mapEnv);
     Class<? extends TezRootInputInitializer> initializerClazz = generateSplitsInClient ? null
         : MRInputAMSplitGenerator.class;
     MRHelpers.addMRInput(mapVertex, mapInputPayload, initializerClazz);
@@ -254,11 +250,7 @@ public class OrderedWordCount extends Configured implements Tool {
         setUserPayload(MRHelpers.createUserPayloadFromConf(iReduceStageConf)),
         2,
         MRHelpers.getReduceResource(iReduceStageConf));
-    ivertex.setJavaOpts(MRHelpers.getReduceJavaOpts(iReduceStageConf));
     ivertex.setTaskLocalResources(commonLocalResources);
-    Map<String, String> ireduceEnv = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(iReduceStageConf, ireduceEnv, false);
-    ivertex.setTaskEnvironment(ireduceEnv);
     vertices.add(ivertex);
 
     byte[] finalReducePayload = MRHelpers.createUserPayloadFromConf(finalReduceConf);
@@ -266,12 +258,7 @@ public class OrderedWordCount extends Configured implements Tool {
         new ProcessorDescriptor(
             ReduceProcessor.class.getName()).setUserPayload(finalReducePayload),
                 1, MRHelpers.getReduceResource(finalReduceConf));
-    finalReduceVertex.setJavaOpts(
-        MRHelpers.getReduceJavaOpts(finalReduceConf));
     finalReduceVertex.setTaskLocalResources(commonLocalResources);
-    Map<String, String> reduceEnv = new HashMap<String, String>();
-    MRHelpers.updateEnvironmentForMRTasks(finalReduceConf, reduceEnv, false);
-    finalReduceVertex.setTaskEnvironment(reduceEnv);
     MRHelpers.addMROutputLegacy(finalReduceVertex, finalReducePayload);
     vertices.add(finalReduceVertex);
 
@@ -360,9 +347,6 @@ public class OrderedWordCount extends Configured implements Tool {
     TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[] {stagingDir}, conf);
     TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
 
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
-        MRHelpers.getMRAMJavaOpts(conf));
-
     // No need to add jar containing this class as assumed to be part of
     // the tez jars.
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
index e108e82..de80455 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
@@ -190,27 +190,22 @@ public class UnionExample {
     Vertex mapVertex1 = new Vertex("map1", new ProcessorDescriptor(
         TokenProcessor.class.getName()),
         numMaps, MRHelpers.getMapResource(tezConf));
-    mapVertex1.setJavaOpts(MRHelpers.getMapJavaOpts(tezConf));
     mapVertex1.addInput("MRInput", id, MRInputAMSplitGenerator.class);
 
     Vertex mapVertex2 = new Vertex("map2", new ProcessorDescriptor(
         TokenProcessor.class.getName()),
         numMaps, MRHelpers.getMapResource(tezConf));
-    mapVertex2.setJavaOpts(MRHelpers.getMapJavaOpts(tezConf));
     mapVertex2.addInput("MRInput", id, MRInputAMSplitGenerator.class);
 
     Vertex mapVertex3 = new Vertex("map3", new ProcessorDescriptor(
         TokenProcessor.class.getName()),
         numMaps, MRHelpers.getMapResource(tezConf));
-    mapVertex3.setJavaOpts(MRHelpers.getMapJavaOpts(tezConf));
     mapVertex3.addInput("MRInput", id, MRInputAMSplitGenerator.class);
 
     Vertex checkerVertex = new Vertex("checker",
         new ProcessorDescriptor(
             UnionProcessor.class.getName()),
                 1, MRHelpers.getReduceResource(tezConf));
-    checkerVertex.setJavaOpts(
-        MRHelpers.getReduceJavaOpts(tezConf));
 
     Configuration outputConf = new Configuration(tezConf);
     outputConf.set(FileOutputFormat.OUTDIR, outputPath);
@@ -300,9 +295,6 @@ public class UnionExample {
     // security
     TokenCache.obtainTokensForNamenodes(credentials, new Path[] {stagingDir}, tezConf);
     TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
- 
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
-        MRHelpers.getMRAMJavaOpts(tezConf));
 
     // No need to add jar containing this class as assumed to be part of
     // the tez jars.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
index 3a6b8e2..10ed4a6 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
@@ -139,14 +139,11 @@ public class WordCount extends Configured implements Tool {
     
     Vertex tokenizerVertex = new Vertex("tokenizer", new ProcessorDescriptor(
         TokenProcessor.class.getName()), -1, MRHelpers.getMapResource(tezConf));
-    tokenizerVertex.setJavaOpts(MRHelpers.getMapJavaOpts(tezConf));
     tokenizerVertex.addInput("MRInput", id, MRInputAMSplitGenerator.class);
 
     Vertex summerVertex = new Vertex("summer",
         new ProcessorDescriptor(
             SumProcessor.class.getName()), 1, MRHelpers.getReduceResource(tezConf));
-    summerVertex.setJavaOpts(
-        MRHelpers.getReduceJavaOpts(tezConf));
     summerVertex.addOutput("MROutput", od, MROutputCommitter.class);
     
     DAG dag = new DAG("WordCount");
@@ -198,9 +195,6 @@ public class WordCount extends Configured implements Tool {
     TokenCache.obtainTokensForNamenodes(credentials, new Path[] {stagingDir}, tezConf);
     TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
 
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
-        MRHelpers.getMRAMJavaOpts(tezConf));
-
     // No need to add jar containing this class as assumed to be part of
     // the tez jars.
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index 49057d3..7c6ba7e 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -587,7 +587,7 @@ public class YARNRunner implements ClientProtocol {
     MRHelpers.updateEnvironmentForMRAM(conf, environment);
 
     TezConfiguration dagAMConf = getDAGAMConfFromMRConf();
-    dagAMConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, javaOpts.toString());
+    dagAMConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, javaOpts.toString());
 
     // Submit to ResourceManager
     try {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 9ad352d..6355430 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -871,7 +871,7 @@ public class MRHelpers {
         Environment.PWD.$(), File.pathSeparator);
 
     // Add the env variables passed by the admin
-    TezYARNUtils.setEnvFromInputString(environment, conf.get(
+    TezYARNUtils.appendToEnvFromInputString(environment, conf.get(
         MRJobConfig.MAPRED_ADMIN_USER_ENV,
         MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
         File.pathSeparator);
@@ -880,7 +880,7 @@ public class MRHelpers {
     String mapredChildEnv = (isMap ?
         conf.get(MRJobConfig.MAP_ENV, "")
         : conf.get(MRJobConfig.REDUCE_ENV, ""));
-    TezYARNUtils.setEnvFromInputString(environment, mapredChildEnv, File.pathSeparator);
+    TezYARNUtils.appendToEnvFromInputString(environment, mapredChildEnv, File.pathSeparator);
 
     // Set logging level in the environment.
     environment.put(
@@ -919,9 +919,9 @@ public class MRHelpers {
    * @param environment Environment map to update
    */
   public static void updateEnvironmentForMRAM(Configuration conf, Map<String, String> environment) {
-    TezYARNUtils.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV),
+    TezYARNUtils.appendToEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV),
         File.pathSeparator);
-    TezYARNUtils.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV),
+    TezYARNUtils.appendToEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV),
         File.pathSeparator);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index d5fd04d..0220a96 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -532,18 +532,10 @@ public class TestMRRJobsDAGApi {
       stage1Vertex.setTaskLocalResources(commonLocalResources);
     }
 
-    stage1Vertex.setJavaOpts(MRHelpers.getMapJavaOpts(stage1Conf));
-    stage1Vertex.setTaskEnvironment(commonEnv);
-
     // TODO env, resources
 
-    stage2Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage2Conf));
     stage2Vertex.setTaskLocalResources(commonLocalResources);
-    stage2Vertex.setTaskEnvironment(commonEnv);
-
-    stage3Vertex.setJavaOpts(MRHelpers.getReduceJavaOpts(stage3Conf));
     stage3Vertex.setTaskLocalResources(commonLocalResources);
-    stage3Vertex.setTaskEnvironment(commonEnv);
 
     dag.addVertex(stage1Vertex);
     dag.addVertex(stage2Vertex);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
index cd54751..b3125f8 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
@@ -131,7 +131,7 @@ public class TestDAGRecovery {
     tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
     tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
     tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, " -Xmx256m");
+    tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
 
     AMConfiguration amConfig = new AMConfiguration(
         new HashMap<String, String>(), new HashMap<String, LocalResource>(),

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/af538639/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
index b1eeea4..413595c 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
@@ -123,7 +123,7 @@ public class TestDAGRecovery2 {
     tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
     tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4);
     tezConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 500);
-    tezConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, " -Xmx256m");
+    tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
     return tezConf;
   }
 


Mime
View raw message