tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject [2/2] git commit: TEZ-219. Create helper api to create environment and java opts for Tez AM. (hitesh)
Date Fri, 14 Jun 2013 22:13:48 GMT
TEZ-219. Create helper api to create environment and java opts for Tez AM. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/e8c31ed6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/e8c31ed6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/e8c31ed6

Branch: refs/heads/master
Commit: e8c31ed679e2bfb8d7884e3e98125f3505f1f0af
Parents: 89e3c0b
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Jun 14 15:09:03 2013 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Jun 14 15:09:03 2013 -0700

----------------------------------------------------------------------
 .../apache/tez/dag/api/TezConfiguration.java    | 44 +++++++--------
 .../apache/tez/mapreduce/hadoop/MRHelpers.java  | 57 ++++++++++++++++++++
 .../tez/mapreduce/hadoop/TestMRHelpers.java     | 29 ++++++++++
 3 files changed, 108 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e8c31ed6/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 498e3fb..6f915be 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -38,79 +38,79 @@ public class TezConfiguration extends Configuration {
 
   public static final String TEZ_PREFIX = "tez.";
   public static final String TEZ_AM_PREFIX = TEZ_PREFIX + "dag.am.";
-  
+
   public static final String DAG_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir";
   public static final String DAG_AM_STAGING_DIR_DEFAULT = "/tmp/hadoop-yarn/staging";
-  
+
   // TODO Should not be required once all tokens are handled via AppSubmissionContext
   public static final String JOB_SUBMIT_DIR = TEZ_PREFIX + "jobSubmitDir";
   public static final String APPLICATION_TOKENS_FILE = "appTokens";
-  public static final String DAG_APPLICATION_MASTER_CLASS = 
+  public static final String DAG_APPLICATION_MASTER_CLASS =
       "org.apache.tez.dag.app.DAGAppMaster";
 
   /** Root Logging level passed to the Tez app master.*/
   public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX+"log.level";
   public static final String DEFAULT_TEZ_AM_LOG_LEVEL = "INFO";
-  
-  public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN = TEZ_AM_PREFIX + 
+
+  public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN = TEZ_AM_PREFIX +
       "am.complete.cancel.delegation.tokens";
   public static final boolean DEFAULT_TEZ_AM_CANCEL_DELEGATION_TOKEN = true;
-  
-  public static final String DAG_AM_TASK_LISTENER_THREAD_COUNT = 
+
+  public static final String DAG_AM_TASK_LISTENER_THREAD_COUNT =
                                 TEZ_PREFIX + "task.listener.thread-count";
   public static final int DAG_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;
 
-  public static final String DAG_AM_CONTAINER_LISTENER_THREAD_COUNT = 
+  public static final String DAG_AM_CONTAINER_LISTENER_THREAD_COUNT =
       TEZ_PREFIX + "container.listener.thread-count";
   public static final int DAG_AM_CONTAINER_LISTENER_THREAD_COUNT_DEFAULT = 30;
-  
+
   // TODO Some of the DAG properties are job specific and not AM specific. Rename accordingly.
   // TODO Are any of these node blacklisting properties required. (other than for MR compat)
   public static final String DAG_MAX_TASK_FAILURES_PER_NODE = TEZ_PREFIX
       + "maxtaskfailures.per.node";
   public static final int DAG_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 3;
-  
-  public static final String DAG_MAX_TASK_ATTEMPTS = 
+
+  public static final String DAG_MAX_TASK_ATTEMPTS =
       TEZ_AM_PREFIX + "max.task.attempts";
   public static final int DAG_MAX_TASK_ATTEMPTS_DEFAULT = 4;
-  
+
   public static final String DAG_NODE_BLACKLISTING_ENABLED = TEZ_PREFIX
       + "node-blacklisting.enabled";
   public static final boolean DAG_NODE_BLACKLISTING_ENABLED_DEFAULT = true;
   public static final String DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD = TEZ_PREFIX
       + "node-blacklisting.ignore-threshold-node-percent";
   public static final int DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33;
-  
+
   /** Number of threads to handle job client RPC requests.*/
   public static final String DAG_CLIENT_AM_THREAD_COUNT =
                                     TEZ_PREFIX + "client.am.thread-count";
   public static final int DAG_CLIENT_AM__THREAD_COUNT_DEFAULT = 1;
-  /** 
+  /**
    * Range of ports that the AM can use when binding. Leave blank
    * if you want all possible ports.
    */
-  public static final String DAG_CLIENT_AM_PORT_RANGE = 
+  public static final String DAG_CLIENT_AM_PORT_RANGE =
                                     TEZ_PREFIX + "client.am.port-range";
 
 
   public static final String DAG_AM_RESOURCE_MEMORY_MB = TEZ_AM_PREFIX
       + "resource.memory.mb";
-  public static final int DEFAULT_DAG_AM_RESOURCE_MEMORY_MB = 1024;
+  public static final int DEFAULT_DAG_AM_RESOURCE_MEMORY_MB = 1536;
 
   public static final String DAG_AM_RESOURCE_CPU_VCORES = TEZ_AM_PREFIX
       + "resource.cpu.vcores";
   public static final int DEFAULT_DAG_AM_RESOURCE_CPU_VCORES = 1;
-  
-  public static final String 
+
+  public static final String
           SLOWSTART_VERTEX_SCHEDULER_MIN_SRC_FRACTION = TEZ_PREFIX
           + "slowstart-vertex-scheduler.min-src-fraction";
-  public static final float 
+  public static final float
           SLOWSTART_VERTEX_SCHEDULER_MIN_SRC_FRACTION_DEFAULT = 1f;
 
-  public static final String 
+  public static final String
           SLOWSTART_VERTEX_SCHEDULER_MAX_SRC_FRACTION = TEZ_PREFIX
           + "slowstart-vertex-scheduler.max-src-fraction";
-  public static final float 
+  public static final float
           SLOWSTART_VERTEX_SCHEDULER_MAX_SRC_FRACTION_DEFAULT = 1f;
 
   /**
@@ -127,6 +127,6 @@ public class TezConfiguration extends Configuration {
 
   public static final String TEZ_LIB_URIS =
       TEZ_PREFIX + "lib.uris";
-  
+
   public static final String TEZ_APPLICATION_TYPE = "TEZ-MR*";
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e8c31ed6/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 475b3a0..b84a2e3 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -24,6 +24,7 @@ import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Vector;
@@ -44,6 +45,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobSubmissionFiles;
 import org.apache.hadoop.mapreduce.split.JobSplitWriter;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.ContainerLogAppender;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -595,4 +597,59 @@ public class MRHelpers {
         getChildLogLevel(conf, isMap) + ",CLA");
   }
 
+  private static Configuration getBaseJobConf(Configuration conf) {
+    if (conf != null) {
+      return new JobConf(conf);
+    } else {
+      return new JobConf();
+    }
+  }
+
+  /**
+   * Get default initialize JobConf-based configuration
+   * @param conf Conf to initialize JobConf with.
+   * @return Base configuration for MR-based jobs
+   */
+  public static Configuration getBaseMRConfiguration(Configuration conf) {
+    return getBaseJobConf(conf);
+  }
+
+  /**
+   * Get default initialize JobConf-based configuration
+   * @return Base configuration for MR-based jobs
+   */
+  public static Configuration getBaseMRConfiguration() {
+    return getBaseJobConf(null);
+  }
+
+  /**
+   * Setup environment for the AM based on MR-based configuration
+   * @param conf Configuration from which to extract information
+   * @param environment Environment map to update
+   */
+  public static void updateEnvironmentForMRAM(Configuration conf,
+      Map<String, String> environment) {
+    MRApps.setEnvFromInputString(environment,
+        conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
+    MRApps.setEnvFromInputString(environment,
+        conf.get(MRJobConfig.MR_AM_ENV));
+  }
+
+  /**
+   * Extract Java Opts for the AM based on MR-based configuration
+   * @param conf Configuration from which to extract information
+   * @return Java opts for the AM
+   */
+  public static String getMRAMJavaOpts(Configuration conf) {
+    // Admin opts
+    String mrAppMasterAdminOptions = conf.get(
+        MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
+        MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
+    // Add AM user command opts
+    String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
+        MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
+
+    return mrAppMasterAdminOptions.trim()
+        + " " + mrAppMasterUserOptions.trim();
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e8c31ed6/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
index 9158280..ac0ed45 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
@@ -357,4 +357,33 @@ public class TestMRHelpers {
     Assert.assertEquals("red1", env.get("foo"));
     Assert.assertEquals("red2", env.get("bar"));
   }
+
+  @Test
+  public void testGetBaseMRConf() {
+    Configuration conf = MRHelpers.getBaseMRConfiguration();
+    Assert.assertNotNull(conf);
+    conf = MRHelpers.getBaseMRConfiguration(new YarnConfiguration());
+    Assert.assertNotNull(conf);
+  }
+
+  @Test
+  public void testMRAMJavaOpts() {
+    Configuration conf = new Configuration();
+    conf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, " -Dadminfoobar   ");
+    conf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "  -Duserfoo  ");
+    String opts = MRHelpers.getMRAMJavaOpts(conf);
+    Assert.assertEquals("-Dadminfoobar -Duserfoo", opts);
+  }
+
+  public void testMRAMEnvironmentSetup() {
+    Configuration conf = new Configuration();
+    conf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "foo=bar,admin1=foo1");
+    conf.set(MRJobConfig.MR_AM_ENV, "foo=bar2,user=foo2");
+    Map<String, String> env =
+        new HashMap<String, String>();
+    MRHelpers.updateEnvironmentForMRAM(conf, env);
+    Assert.assertEquals("foo1", env.get("admin1"));
+    Assert.assertEquals("foo2", env.get("user"));
+    Assert.assertEquals("bar2", env.get("foo"));
+  }
 }


Mime
View raw message