tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhiyu...@apache.org
Subject [26/50] [abbrv] tez git commit: TEZ-3647. Add a setting which lets Tez determine Xmx. Contributed by Zhiyuan Yang.
Date Tue, 28 Mar 2017 19:48:07 GMT
TEZ-3647. Add a setting which lets Tez determine Xmx. Contributed by
Zhiyuan Yang.


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1b1eb1d4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1b1eb1d4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1b1eb1d4

Branch: refs/heads/TEZ-1190
Commit: 1b1eb1d4b995c221d2eece2c12fca6ca52b94f27
Parents: 4ce6ea6
Author: Siddharth Seth <sseth@apache.org>
Authored: Fri Mar 3 13:45:05 2017 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Fri Mar 3 13:45:05 2017 -0800

----------------------------------------------------------------------
 CHANGES.txt                                               |  1 +
 .../main/java/org/apache/tez/client/TezClientUtils.java   | 10 +++++++++-
 .../java/org/apache/tez/dag/api/TezConfiguration.java     |  4 ++++
 .../main/java/org/apache/tez/dag/api/TezConstants.java    |  6 ++++++
 .../java/org/apache/tez/client/TestTezClientUtils.java    | 10 ++++++++++
 5 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 07841bf..763acd8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3647. Add a setting which lets Tez determine Xmx.
   TEZ-3644. Cleanup container list stored in AMNode.
   TEZ-3646. IFile.Writer has an extra output stream flush call
   TEZ-3643. Long running AMs can go out of memory due to retained AMContainer instances.

http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index ab37b17..b16d1ca 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -39,6 +39,7 @@ import java.util.Map.Entry;
 
 import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Precision;
 import org.apache.tez.common.JavaOptsChecker;
 import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
 import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
@@ -963,9 +964,16 @@ public class TezClientUtils {
         || (resource.getMemory() <= 0)) {
       return javaOpts;
     }
-    if (maxHeapFactor <= 0 || maxHeapFactor >= 1) {
+
+    if ((maxHeapFactor <= 0 && !Precision.equals(maxHeapFactor, -1, 0.01)) ||
maxHeapFactor >= 1) {
       return javaOpts;
     }
+
+    if (Precision.equals(maxHeapFactor, -1, 0.01)) {
+      maxHeapFactor = resource.getMemory() < TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB
+        ? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB
+        : TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB;
+    }
     int maxMemory = (int)(resource.getMemory() * maxHeapFactor);
     maxMemory = maxMemory <= 0 ? 1 : maxMemory;
 

http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 94f40bb..5900f5e 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -356,6 +356,10 @@ public class TezConfiguration extends Configuration {
    * actual allocation of memory to tasks the cluster. The value if used as a
    * fraction that is applied to the memory allocated Factor to size Xmx based
    * on container memory size. Value should be greater than 0 and less than 1.
+   *
+   * Set this value to -1 to allow Tez to use different default max heap fraction
+   * for different container memory size. Current policy is to use 0.7 for container
+   * smaller than 4GB and use 0.8 for larger container.
    */
   @ConfigurationScope(Scope.AM)
   @ConfigurationProperty(type="float")

http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index 06b9cb7..b75269a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -123,5 +123,11 @@ public class TezConstants {
    */
   public static final long TEZ_AM_CLIENT_HEARTBEAT_POLL_INTERVAL_MILLIS_MINIMUM = 1000;
 
+  /**
+   * For container JVM Xmx auto determination
+   */
+  public static final int TEZ_CONTAINER_SMALL_SLAB_BOUND_MB = 4096;
+  public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB = 0.7;
+  public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB = 0.8;
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index a5e9d3c..7ff8125 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -629,6 +629,16 @@ public class TestTezClientUtils {
     javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
         Resource.newInstance(355, 1), 100);
     Assert.assertEquals(origJavaOpts, javaOpts);
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(1000, 1), -1);
+    Assert.assertTrue(javaOpts.contains("-Xmx700m"));
+
+    origJavaOpts = "";
+    javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+        Resource.newInstance(5000, 1), -1);
+    Assert.assertTrue(javaOpts.contains("-Xmx4000m"));
   }
 
   @Test (timeout=5000)


Mime
View raw message