apex-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pra...@apache.org
Subject [apex-core] branch master updated: APEXCORE-779 In unit tests Yarn containers must use the same JVM as the test itself.
Date Tue, 29 Aug 2017 23:30:48 GMT
This is an automated email from the ASF dual-hosted git repository.

pramod pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apex-core.git


The following commit(s) were added to refs/heads/master by this push:
     new 821184e  APEXCORE-779 In unit tests Yarn containers must use the same JVM as the
test itself.
821184e is described below

commit 821184e296d2ecc982c3b3275d28c7167d0768e8
Author: Vlad Rozov <vrozov@apache.org>
AuthorDate: Sat Aug 26 10:03:45 2017 -0700

    APEXCORE-779 In unit tests Yarn containers must use the same JVM as the test itself.
---
 .../java/com/datatorrent/stram/StramClient.java    |  3 ++-
 .../datatorrent/stram/StramMiniClusterTest.java    | 30 +++-------------------
 2 files changed, 6 insertions(+), 27 deletions(-)

diff --git a/engine/src/main/java/com/datatorrent/stram/StramClient.java b/engine/src/main/java/com/datatorrent/stram/StramClient.java
index 06abdae..3a3c135 100644
--- a/engine/src/main/java/com/datatorrent/stram/StramClient.java
+++ b/engine/src/main/java/com/datatorrent/stram/StramClient.java
@@ -103,6 +103,8 @@ public class StramClient
   @Deprecated
   public static final String YARN_APPLICATION_TYPE_DEPRECATED = "DataTorrent";
 
+  private static final String javaCmd = "${JAVA_HOME}" + "/bin/java";
+
   public static final String LIB_JARS_SEP = ",";
 
   // Configuration
@@ -114,7 +116,6 @@ public class StramClient
   private final int amPriority = 0;
   private ApplicationId appId;
   private final LogicalPlan dag;
-  public String javaCmd = "${JAVA_HOME}" + "/bin/java";
   // log4j.properties file
   // if available, add to local resources and set into classpath
   private final String log4jPropFile = "";
diff --git a/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java b/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java
index 455604b..0e3c0ab 100644
--- a/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java
@@ -44,7 +44,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.apex.common.util.JarHelper;
-import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -125,19 +124,10 @@ public class StramMiniClusterTest
     conf = StramClientUtils.addDTDefaultResources(conf);
     conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
     conf.setInt("yarn.nodemanager.vmem-pmem-ratio", 20); // workaround to avoid containers
being killed because java allocated too much vmem
-    conf.setStrings("yarn.scheduler.capacity.root.queues", "default");
-    conf.setStrings("yarn.scheduler.capacity.root.default.capacity", "100");
-
-    StringBuilder adminEnv = new StringBuilder(1024);
-    if (System.getenv("JAVA_HOME") == null) {
-      adminEnv.append("JAVA_HOME=").append(System.getProperty("java.home"));
-      adminEnv.append(",");
-    }
-    adminEnv.append("MALLOC_ARENA_MAX=4"); // see MAPREDUCE-3068, MAPREDUCE-3065
-    adminEnv.append(",");
-    adminEnv.append("CLASSPATH=").append(getTestRuntimeClasspath());
-
-    conf.set(YarnConfiguration.NM_ADMIN_USER_ENV, adminEnv.toString());
+    conf.set("yarn.scheduler.capacity.root.queues", "default");
+    conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+    conf.set(YarnConfiguration.NM_ADMIN_USER_ENV, String.format("JAVA_HOME=%s,CLASSPATH=%s",
System.getProperty("java.home"), getTestRuntimeClasspath()));
+    conf.set(YarnConfiguration.NM_ENV_WHITELIST, YarnConfiguration.DEFAULT_NM_ENV_WHITELIST.replaceAll("JAVA_HOME,*",
""));
 
     if (yarnCluster == null) {
       yarnCluster = new MiniYARNCluster(StramMiniClusterTest.class.getName(),
@@ -229,9 +219,6 @@ public class StramMiniClusterTest
     StramClient client = new StramClient(yarnConf, dag);
     try {
       client.start();
-      if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
-        client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
-      }
       LOG.info("Running client");
       client.startApplication();
       boolean result = client.monitorApplication();
@@ -275,9 +262,6 @@ public class StramMiniClusterTest
     tb.addFromProperties(props, null);
 
     StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb));
-    if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
-      client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
-    }
     try {
       client.start();
       client.startApplication();
@@ -382,9 +366,6 @@ public class StramMiniClusterTest
 
     LOG.info("Initializing Client");
     StramClient client = new StramClient(conf, dag);
-    if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
-      client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
-    }
     try {
       client.start();
       client.startApplication();
@@ -614,9 +595,6 @@ public class StramMiniClusterTest
     dag.getContextAttributes(operator).put(OperatorContext.RECOVERY_ATTEMPTS, 0);
 
     StramClient client = new StramClient(conf, dag);
-    if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
-      client.javaCmd = "java";
-    }
     try {
       client.start();
       client.startApplication();

-- 
To stop receiving notification emails like this one, please contact
['"commits@apex.apache.org" <commits@apex.apache.org>'].

Mime
View raw message