tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject [2/2] git commit: TEZ-218. Create helper api to define environment and classpath for vertices. (hitesh)
Date Thu, 13 Jun 2013 23:02:12 GMT
TEZ-218. Create helper api to define environment and classpath for vertices. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8194c0dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8194c0dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8194c0dd

Branch: refs/heads/master
Commit: 8194c0ddec484868342f184261ea55aa77cd6035
Parents: e73ff02
Author: Hitesh Shah <hitesh@apache.org>
Authored: Thu Jun 13 16:01:53 2013 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Thu Jun 13 16:01:53 2013 -0700

----------------------------------------------------------------------
 .../apache/tez/mapreduce/hadoop/MRHelpers.java  | 61 +++++++++++++++++++-
 .../tez/mapreduce/hadoop/TestMRHelpers.java     | 47 ++++++++++++++-
 2 files changed, 104 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8194c0dd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 8d5acbd..4a9666c 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -46,11 +46,13 @@ import org.apache.hadoop.mapreduce.split.JobSplitWriter;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.ContainerLogAppender;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 
@@ -334,7 +336,7 @@ public class MRHelpers {
    * Sets up parameters which used to be set by the MR JobClient. Includes
    * setting whether to use the new api or the old api. Note: Must be called
    * before generating InputSplits
-   * 
+   *
    * @param conf
    *          configuration for the vertex.
    */
@@ -371,11 +373,11 @@ public class MRHelpers {
       }
     }
   }
-  
+
   /**
    * Default to the new APIs unless they are explicitly set or the old mapper or
    * reduce attributes are used.
-   * 
+   *
    * @throws IOException
    *           if the configuration is inconsistant
    */
@@ -531,4 +533,57 @@ public class MRHelpers {
             MRJobConfig.DEFAULT_REDUCE_CPU_VCORES));
   }
 
+  /**
+   * Setup classpath and other environment variables
+   * @param conf Configuration to retrieve settings from
+   * @param environment Environment to update
+   * @param isMap Whether task is a map or reduce task
+   */
+  public static void updateEnvironmentForMRTasks(Configuration conf,
+      Map<String, String> environment, boolean isMap) {
+
+    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+          System.getProperty("java.class.path"));
+    }
+
+    // TEZ jars and deps will be localized by the TezClient submission layer
+    // Assumption is that MR is also in TEZ dependencies
+    Apps.addToEnvironment(environment,
+        Environment.CLASSPATH.name(),
+        Environment.PWD.$());
+
+    // Add YARN/COMMON/HDFS jars to path
+    for (String c : conf.getStrings(
+        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+          c.trim());
+    }
+
+    // Shell
+    environment.put(Environment.SHELL.name(), conf.get(
+        MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
+
+    // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
+    Apps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
+        Environment.PWD.$());
+
+    // Add the env variables passed by the admin
+    Apps.setEnvFromInputString(environment, conf.get(
+        MRJobConfig.MAPRED_ADMIN_USER_ENV,
+        MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV));
+
+    // Add the env variables passed by the user
+    String mapredChildEnv = (isMap ?
+        conf.get(MRJobConfig.MAP_ENV, "")
+        : conf.get(MRJobConfig.REDUCE_ENV, ""));
+    Apps.setEnvFromInputString(environment, mapredChildEnv);
+
+    // Set logging level in the environment.
+    environment.put(
+        "HADOOP_ROOT_LOGGER",
+        getChildLogLevel(conf, isMap) + ",CLA");
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8194c0dd/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
index 2b34227..0b03fd3 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
 import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -286,7 +287,7 @@ public class TestMRHelpers {
 
   @Test
   public void testContainerResourceConstruction() {
-    Configuration conf = new Configuration();
+    JobConf conf = new JobConf(new Configuration());
     Resource mapResource = MRHelpers.getMapResource(conf);
     Resource reduceResource = MRHelpers.getReduceResource(conf);
 
@@ -312,4 +313,48 @@ public class TestMRHelpers {
     Assert.assertEquals(20, reduceResource.getVirtualCores());
     Assert.assertEquals(1234, reduceResource.getMemory());
   }
+
+  private Configuration setupConfigForMREnvTest() {
+    JobConf conf = new JobConf(new Configuration());
+    conf.set(MRJobConfig.MAP_ENV, "foo=map1,bar=map2");
+    conf.set(MRJobConfig.REDUCE_ENV, "foo=red1,bar=red2");
+    conf.set(MRJobConfig.MAP_LOG_LEVEL, "TRACE");
+    conf.set(MRJobConfig.REDUCE_LOG_LEVEL, "FATAL");
+    return conf;
+  }
+
+  private void testCommonEnvSettingsForMRTasks(Map<String, String> env) {
+    Assert.assertTrue(env.containsKey("foo"));
+    Assert.assertTrue(env.containsKey("bar"));
+    Assert.assertTrue(env.containsKey(Environment.CLASSPATH.$()));
+    Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.$()));
+    Assert.assertTrue(env.containsKey(Environment.SHELL.$()));
+    Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));
+    for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
+      Assert.assertTrue(env.get(Environment.CLASSPATH.$()).contains(val));
+    }
+
+    Assert.assertTrue(0 ==
+        env.get(Environment.CLASSPATH.$()).indexOf(Environment.PWD.$()));
+
+  }
+  @Test
+  public void testMREnvSetupForMap() {
+    Configuration conf = setupConfigForMREnvTest();
+    Map<String, String> env = new HashMap<String, String>();
+    MRHelpers.updateEnvironmentForMRTasks(conf, env, true);
+    testCommonEnvSettingsForMRTasks(env);
+    Assert.assertEquals("map1", env.get("foo"));
+    Assert.assertEquals("map2", env.get("bar"));
+  }
+
+  @Test
+  public void testMREnvSetupForReduce() {
+    Configuration conf = setupConfigForMREnvTest();
+    Map<String, String> env = new HashMap<String, String>();
+    MRHelpers.updateEnvironmentForMRTasks(conf, env, false);
+    testCommonEnvSettingsForMRTasks(env);
+    Assert.assertEquals("red1", env.get("foo"));
+    Assert.assertEquals("red2", env.get("bar"));
+  }
 }


Mime
View raw message