tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject git commit: TEZ-699. Have sensible defaults for java opts. (hitesh)
Date Fri, 27 Jun 2014 23:18:24 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master df7ff3cb4 -> 7aa927a72


TEZ-699. Have sensible defaults for java opts. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/7aa927a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/7aa927a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/7aa927a7

Branch: refs/heads/master
Commit: 7aa927a72f8142a5d6e05531fd948dbff85493a9
Parents: df7ff3c
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Jun 27 16:17:55 2014 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Jun 27 16:17:55 2014 -0700

----------------------------------------------------------------------
 .../org/apache/tez/client/TezClientUtils.java   | 34 +++++++++--
 .../main/java/org/apache/tez/dag/api/DAG.java   |  6 +-
 .../apache/tez/dag/api/TezConfiguration.java    | 15 +++--
 .../apache/tez/client/TestTezClientUtils.java   | 59 ++++++++++++++++++++
 .../app/rm/container/AMContainerHelpers.java    | 22 ++++++--
 .../dag/app/rm/container/AMContainerImpl.java   |  3 +-
 .../dag/app/rm/container/TestAMContainer.java   |  3 +
 7 files changed, 125 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index bae0328..dd1b997 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -314,7 +314,7 @@ public class TezClientUtils {
    * @param amName Name for the application
    * @param amConfig AM Configuration
    * @param tezJarResources Resources to be used by the AM
-   * @param sessionCredentials the credential object which will be populated with session
specific
+   * @param sessionCreds the credential object which will be populated with session specific
    * @return an ApplicationSubmissionContext to launch a Tez AM
    * @throws IOException
    * @throws YarnException
@@ -377,14 +377,14 @@ public class TezClientUtils {
 
     String amOpts = amConfig.getAMConf().get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
         TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);
-    if (amOpts != null && !amOpts.isEmpty()) {
-      vargs.add(amOpts);
-    }
+    amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
+        amConfig.getAMConf().getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
+            TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
+    vargs.add(amOpts);
 
     String amLogLevel = amConfig.getAMConf().get(
         TezConfiguration.TEZ_AM_LOG_LEVEL,
         TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
-
     maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs);
 
     // FIX sun bug mentioned in TEZ-327
@@ -805,4 +805,28 @@ public class TezClientUtils {
     TokenCache.setSessionToken(sessionToken, credentials);
   }
 
+  /**
+   * Add computed Xmx value to java opts if both -Xms and -Xmx are not specified
+   * @param javaOpts Current java opts
+   * @param resource Resource capability based on which java opts will be computed
+   * @param maxHeapFactor Factor to size Xmx ( valid range is 0.0 < x < 1.0)
+   * @return Modified java opts with computed Xmx value
+   */
+  public static String maybeAddDefaultMemoryJavaOpts(String javaOpts, Resource resource,
+      double maxHeapFactor) {
+    if ((javaOpts != null && !javaOpts.isEmpty()
+          && (javaOpts.contains("-Xmx") || javaOpts.contains("-Xms")))
+        || (resource.getMemory() <= 0)) {
+      return javaOpts;
+    }
+    if (maxHeapFactor <= 0 || maxHeapFactor >= 1) {
+      return javaOpts;
+    }
+    int maxMemory = (int)(resource.getMemory() * maxHeapFactor);
+    maxMemory = maxMemory <= 0 ? 1 : maxMemory;
+
+    return " -Xmx" + maxMemory + "m "
+        + ( javaOpts != null ? javaOpts : "");
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 9df37b3..4772fae 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -62,7 +62,7 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
-public class DAG { // FIXME rename to Topology
+public class DAG {
   
   private static final Log LOG = LogFactory.getLog(DAG.class);
   
@@ -498,9 +498,9 @@ public class DAG { // FIXME rename to Topology
         // there was something on the stack other than this "av".
         // this indicates there is a scc/cycle. It comprises all nodes from top of stack
to "av"
         StringBuilder message = new StringBuilder();
-        message.append(av.v.getVertexName() + " <- ");
+        message.append(av.v.getVertexName()).append(" <- ");
         for (; pop != av; pop = stack.pop()) {
-          message.append(pop.v.getVertexName() + " <- ");
+          message.append(pop.v.getVertexName()).append(" <- ");
           pop.onstack = false;
         }
         message.append(av.v.getVertexName());

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 8c1bf8a..610f919 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -66,16 +66,22 @@ public class TezConfiguration extends Configuration {
   public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX +  "java.opts";
   public static final String TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT = 
       "-Djava.net.preferIPv4Stack=true " +
-      "-Dhadoop.metrics.log.level=WARN " + 
-      "-Xmx1024m"; // Remove after TEZ-699
+      "-Dhadoop.metrics.log.level=WARN ";
 
   /** Command line options for the Tez Task processes. */
   public static final String TEZ_TASK_LAUNCH_CMD_OPTS = TEZ_TASK_PREFIX
       + "launch.cmd-opts";
   public static final String TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT = 
       "-Djava.net.preferIPv4Stack=true " +
-      "-Dhadoop.metrics.log.level=WARN " + 
-      "-Xmx200m"; // Remove after TEZ-699
+      "-Dhadoop.metrics.log.level=WARN ";
+
+  /**
+   * Factor to size Xmx based on container memory size. Value should be greater than 0 and
+   * less than 1.
+   */
+  public static final String TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION =
+      TEZ_PREFIX + "container.max.java.heap.fraction";
+  public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT = 0.8;
 
   /** Env settings for the Tez AppMaster process.
    * Should be specified as a comma-separated of key-value pairs where each pair
@@ -478,4 +484,5 @@ public class TezConfiguration extends Configuration {
    * The maximium number of tasks running in parallel in inline mode. Not valid till Tez-684
get checked-in
    */
   public static final int TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT = 1;
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 1762a8a..e519187 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -19,6 +19,7 @@ package org.apache.tez.client;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -31,6 +32,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.junit.Assert;
@@ -61,6 +63,7 @@ public class TestTezClientUtils {
   @Test (timeout=5000)
   public void validateSetTezJarLocalResourcesDefinedButEmpty() throws Exception {
     File emptyDir = new File(TEST_ROOT_DIR, "emptyDir");
+    emptyDir.deleteOnExit();
     Assert.assertTrue(emptyDir.mkdirs());
     Path emptyDirPath = new Path(emptyDir.getAbsolutePath());
     TezConfiguration conf = new TezConfiguration();
@@ -159,4 +162,60 @@ public class TestTezClientUtils {
     Map<String, LocalResource> localizedMap = TezClientUtils.setupTezJarsLocalResources(conf,
credentials);
     assertFalse(localizedMap.isEmpty());
   }
+
+
+  @Test (timeout=5000)
+  public void testDefaultMemoryJavaOpts() {
+    final double factor = 0.8;
+    String origJavaOpts = "-Xmx";
+    String javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(1000, 1), factor);
+    Assert.assertEquals(origJavaOpts, javaOpts);
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(1000, 1), factor);
+    Assert.assertTrue(javaOpts.contains("-Xmx800m"));
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(1, 1), factor);
+    Assert.assertTrue(javaOpts.contains("-Xmx1m"));
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(-1, 1), factor);
+    Assert.assertEquals(origJavaOpts, javaOpts);
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(355, 1), factor);
+    Assert.assertTrue(javaOpts.contains("-Xmx284m"));
+
+    origJavaOpts = " -Xms100m ";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(355, 1), factor);
+    Assert.assertFalse(javaOpts.contains("-Xmx284m"));
+    Assert.assertTrue(javaOpts.contains("-Xms100m"));
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(355, 1), 0);
+    Assert.assertEquals(origJavaOpts, javaOpts);
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(355, 1), 100);
+    Assert.assertEquals(origJavaOpts, javaOpts);
+  }
+
+  @Test (timeout=5000)
+  public void testDefaultLoggingJavaOpts() {
+    String origJavaOpts = null;
+    String javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts("FOOBAR", origJavaOpts);
+    Assert.assertNotNull(javaOpts);
+    Assert.assertTrue(javaOpts.contains("-D" + TezConfiguration.TEZ_ROOT_LOGGER_NAME + "=FOOBAR")
+        && javaOpts.contains(TezConfiguration.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE));
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index b569fbd..db776a4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -29,6 +29,7 @@ import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -41,8 +42,10 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.TezClientUtils;
 import org.apache.tez.common.security.JobTokenIdentifier;
 import org.apache.tez.common.security.TokenCache;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -98,8 +101,6 @@ public class AMContainerHelpers {
     // Service data
     Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
 
-
-
     // Tokens
     
     // Setup up task credentials buffer
@@ -145,7 +146,8 @@ public class AMContainerHelpers {
       Map<String, String> vertexEnv,
       String javaOpts,
       InetSocketAddress taskAttemptListenerAddress, Credentials credentials,
-      AppContext appContext) {
+      AppContext appContext, Resource containerResource,
+      Configuration conf) {
 
     ContainerLaunchContext commonContainerSpec = null;
     synchronized (commonContainerSpecLock) {
@@ -180,10 +182,22 @@ public class AMContainerHelpers {
     myEnv.putAll(env);
     myEnv.putAll(vertexEnv);
 
+    String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts,
+        containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
+            TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
+    if (LOG.isDebugEnabled()) {
+      if (!modifiedJavaOpts.equals(javaOpts)) {
+        LOG.debug("Modified java opts for container"
+          + ", containerId=" + containerId
+          + ", originalJavaOpts=" + javaOpts
+          + ", modifiedJavaOpts=" + modifiedJavaOpts);
+      }
+    }
+
     List<String> commands = TezRuntimeChildJVM.getVMCommand(
         taskAttemptListenerAddress, containerId.toString(),
         appContext.getApplicationID().toString(),
-        appContext.getApplicationAttemptId().getAttemptId(), javaOpts);
+        appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts);
 
     // Duplicate the ByteBuffers for access by multiple containers.
     Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index ae57658..9d36c58 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -504,7 +504,8 @@ public class AMContainerImpl implements AMContainer {
           containerContext.getEnvironment(),
           containerContext.getJavaOpts(),
           container.taskAttemptListener.getAddress(), containerContext.getCredentials(),
-          container.appContext);
+          container.appContext, container.container.getResource(),
+          container.appContext.getAMConf());
 
       // Registering now, so that in case of delayed NM response, the child
       // task is not told to die since the TAL does not know about the container.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 0862116..3b3fd37 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent;
 import org.apache.hadoop.security.Credentials;
@@ -1039,6 +1040,7 @@ public class TestAMContainer {
       eventHandler = mock(EventHandler.class);
       historyEventHandler = mock(HistoryEventHandler.class);
 
+      Configuration conf = new Configuration(false);
       appContext = mock(AppContext.class);
       doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext)
       .getApplicationACLs();
@@ -1047,6 +1049,7 @@ public class TestAMContainer {
       doReturn(applicationID).when(appContext).getApplicationID();
       doReturn(new SystemClock()).when(appContext).getClock();
       doReturn(historyEventHandler).when(appContext).getHistoryHandler();
+      doReturn(conf).when(appContext).getAMConf();
       mockDAGID();
 
       taskSpec = mock(TaskSpec.class);


Mime
View raw message