tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1710. Add support for cluster default AM/task launch opts. (sseth)
Date Tue, 28 Oct 2014 18:54:08 GMT
Repository: tez
Updated Branches:
  refs/heads/master e225b5a4c -> bc62d81a7


TEZ-1710. Add support for cluster default AM/task launch opts. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bc62d81a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bc62d81a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bc62d81a

Branch: refs/heads/master
Commit: bc62d81a78213175464658f58658f89105a027ad
Parents: e225b5a
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Oct 28 11:53:40 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Oct 28 11:53:40 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/client/TezClientUtils.java   | 39 ++++++---
 .../apache/tez/dag/api/TezConfiguration.java    | 32 +++++--
 .../apache/tez/client/TestTezClientUtils.java   | 92 ++++++++++++++++++++
 4 files changed, 147 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/bc62d81a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9b9c24f..08dfaa5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -67,6 +67,7 @@ ALL CHANGES:
   TEZ-1638. Fix Missing type parametrization in runtime Input/Output configs.
   TEZ-1596. Secure Shuffle utils is extremely expensive for fast queries.
   TEZ-1712. SSL context gets destroyed too soon after downloading data from one of the vertices.
+  TEZ-1710. Add support for cluster default AM/task launch opts.
 
 Release 0.5.1: 2014-10-02
 

http://git-wip-us.apache.org/repos/asf/tez/blob/bc62d81a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 0ac3a26..a917853 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -27,7 +27,6 @@ import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -454,12 +453,7 @@ public class TezClientUtils {
     List<String> vargs = new ArrayList<String>(8);
     vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
 
-    String amOpts = amConfig.getTezConfiguration().get(
-        TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
-        TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);
-    amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
-        amConfig.getTezConfiguration().getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
-            TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
+    String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability);
     vargs.add(amOpts);
 
     String amLogLevel = amConfig.getTezConfiguration().get(
@@ -671,10 +665,17 @@ public class TezClientUtils {
   
   @Private
   public static String addDefaultsToTaskLaunchCmdOpts(String vOpts, Configuration conf) {
-    String vConfigOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS,
+    String vConfigOpts = "";
+    String taskDefaultOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
+        TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
+    if (taskDefaultOpts != null && !taskDefaultOpts.isEmpty()) {
+      vConfigOpts = taskDefaultOpts + " ";
+    }
+    vConfigOpts = vConfigOpts + conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS,
         TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT);
-    if (vConfigOpts != null && vConfigOpts.length() > 0) {
-      vOpts += (" " + vConfigOpts);
+    if (vConfigOpts != null && !vConfigOpts.isEmpty()) {
+      // Add options specified in the DAG at the end.
+      vOpts = vConfigOpts + " " + vOpts;
     }
     
     vOpts = maybeAddDefaultLoggingJavaOpts(conf.get(
@@ -886,4 +887,22 @@ public class TezClientUtils {
     return true;
   }
 
+  @Private
+  @VisibleForTesting
+  static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capability) {
+    String defaultOpts = tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
+        TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
+    String amOpts = "";
+    if (defaultOpts != null && !defaultOpts.isEmpty()) {
+      amOpts = defaultOpts + " ";
+    }
+    amOpts = amOpts + tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
+        TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);
+
+    amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
+        tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
+            TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
+    return amOpts;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/bc62d81a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 5bb565b..492d57f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -140,15 +140,35 @@ public class TezConfiguration extends Configuration {
   public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true;
 
   /**
+   * String value. Command line options which will be prepended to {@link #TEZ_AM_LAUNCH_CMD_OPTS}
+   * during the launch of the AppMaster process. This property will typically be configured
to
+   * include default options meant to be used by all jobs in a cluster. If required, the
values can
+   * be overridden per job.
+   */
+  public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS =
+      TEZ_AM_PREFIX + "launch.cluster-default.admin.cmd-opts";
+  public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT =
+      "-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN";
+
+  /**
    * String value. Command line options provided during the launch of the Tez
    * AppMaster process. Its recommended to not set any Xmx or Xms in these launch opts so
that
    * Tez can determine them automatically.
    * */
   public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX +  "launch.cmd-opts";
   public static final String TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT = 
-      "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc " + 
-      "-XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC " +
-      "-Dhadoop.metrics.log.level=WARN ";
+      "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC";
+
+  /**
+   * String value. Command line options which will be prepended to {@link
+   * #TEZ_TASK_LAUNCH_CMD_OPTS} during the launch of Tez tasks.  This property will typically
be configured to
+   * include default options meant to be used by all jobs in a cluster. If required, the
values can
+   * be overridden per job.
+   */
+  public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS =
+      TEZ_TASK_PREFIX + "launch.cluster-default.cmd-opts";
+  public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT =
+      "-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN";
 
   /**
    * String value. Command line options provided during the launch of Tez Task
@@ -157,10 +177,8 @@ public class TezConfiguration extends Configuration {
    */
   public static final String TEZ_TASK_LAUNCH_CMD_OPTS = TEZ_TASK_PREFIX
       + "launch.cmd-opts";
-  public static final String TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT = 
-      "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc " + 
-      "-XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC " +
-      "-Dhadoop.metrics.log.level=WARN ";
+  public static final String TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT =
+      "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC";
 
   /**
    * Double value. Tez automatically determines the Xmx for the JVMs used to run

http://git-wip-us.apache.org/repos/asf/tez/blob/bc62d81a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 9b764ed..f39c4ea 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -184,6 +184,98 @@ public class TestTezClientUtils {
   }
 
 
+  @Test(timeout = 5000)
+  public void testAMCommandOpts() {
+    TezConfiguration tezConf = new TezConfiguration();
+    String amCommandOpts = "-Xmx 200m -Dtest.property";
+    tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, amCommandOpts);
+
+    // Test1: Rely on defaults for cluster-default opts
+    String amOptsConstructed =
+        TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
+    assertEquals(
+        TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + amCommandOpts,
+        amOptsConstructed);
+
+    // Test2: Setup cluster-default command opts explicitly
+    String clusterDefaultCommandOpts =
+        "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc ";
+    tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, clusterDefaultCommandOpts);
+    amOptsConstructed =
+        TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
+    assertEquals(clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed);
+
+
+    // Test3: Don't setup Xmx explicitly
+    final double factor = 0.8;
+    amCommandOpts = "-Dtest.property";
+    tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, amCommandOpts);
+    amOptsConstructed =
+        TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
+    // It's OK for the Xmx value to show up before cluster default options, since Xmx will
not be replaced if it already exists.
+    assertEquals(
+        " -Xmx" + ((int) (1024 * factor)) + "m" + " " + clusterDefaultCommandOpts + " " +
+            amCommandOpts,
+        amOptsConstructed);
+
+    // Test4: Ensure admin options with Xmx does not cause them to be overridden. This should
almost never be done though.
+    clusterDefaultCommandOpts =
+        "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc -Xmx200m";
+    tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, clusterDefaultCommandOpts);
+    amOptsConstructed =
+        TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
+    assertEquals(clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed);
+  }
+
+  @Test(timeout = 5000)
+  public void testTaskCommandOpts() {
+    TezConfiguration tezConf = new TezConfiguration();
+    String taskCommandOpts = "-Xmx 200m -Dtest.property";
+    tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskCommandOpts);
+    String expected = null;
+
+    // Test1: Rely on defaults for cluster default opts
+    String taskOptsConstructed = TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf);
+    expected =
+        TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + taskCommandOpts;
+    assertTrue(
+        "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed
+
+            "]", taskOptsConstructed.startsWith(expected));
+
+    // Test2: Setup cluster-default command opts explicitly
+    String taskClusterDefaultCommandOpts =
+        "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc ";
+    tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
+        taskClusterDefaultCommandOpts);
+    taskOptsConstructed =
+        TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf);
+    expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts;
+    assertTrue(
+        "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed
+
+            "]", taskOptsConstructed.startsWith(expected));
+
+    // Test3: Don't setup Xmx explicitly
+    taskCommandOpts = "-Dtest.property";
+    tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskCommandOpts);
+    taskOptsConstructed =
+        TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf);
+    expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts;
+    assertTrue(
+        "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed
+
+            "]", taskOptsConstructed.startsWith(expected));
+
+    // Test4: Pass in a dag-configured value.
+    String programmaticTaskOpts = "-Dset.programatically=true -Djava.net.preferIPv4Stack=false";
+    taskOptsConstructed =
+        TezClientUtils.addDefaultsToTaskLaunchCmdOpts(programmaticTaskOpts, tezConf);
+    // Container logging is always added at the end, if it's required.
+    expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts + " " + programmaticTaskOpts;
+    assertTrue(
+        "Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed
+
+            "]", taskOptsConstructed.startsWith(expected));
+  }
+
+
   @Test (timeout=5000)
   public void testDefaultMemoryJavaOpts() {
     final double factor = 0.8;


Mime
View raw message