tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-2662. Provide a way to check whether AM or task opts are valid and error if not. (hitesh)
Date Fri, 28 Aug 2015 02:15:54 GMT
Repository: tez
Updated Branches:
  refs/heads/master 6098f1bb9 -> fd714c296


TEZ-2662. Provide a way to check whether AM or task opts are valid and error if not. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/fd714c29
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/fd714c29
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/fd714c29

Branch: refs/heads/master
Commit: fd714c296c1e33ffcdb6763ab1b67b1312f52e7a
Parents: 6098f1b
Author: Hitesh Shah <hitesh@apache.org>
Authored: Thu Aug 27 19:15:34 2015 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Thu Aug 27 19:15:34 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 .../java/org/apache/tez/client/TezClient.java   |  31 +++++-
 .../org/apache/tez/client/TezClientUtils.java   |  41 +++++--
 .../org/apache/tez/common/JavaOptsChecker.java  |  87 +++++++++++++++
 .../main/java/org/apache/tez/dag/api/DAG.java   |  16 ++-
 .../apache/tez/dag/api/TezConfiguration.java    |  22 ++++
 .../org/apache/tez/client/TestTezClient.java    |  22 ++++
 .../apache/tez/client/TestTezClientUtils.java   |  64 ++++++++++-
 .../apache/tez/common/TestJavaOptsChecker.java  | 111 +++++++++++++++++++
 .../org/apache/tez/dag/api/TestDAGPlan.java     |  57 ++++++++--
 10 files changed, 423 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0c15c1f..82fe016 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
   TEZ-2747. Update master to reflect 0.8.0-alpha release.
+  TEZ-2662. Provide a way to check whether AM or task opts are valid and error if not.
 
 Release 0.8.0-alpha: 2015-08-29 
 
@@ -150,6 +151,7 @@ Release 0.7.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2662. Provide a way to check whether AM or task opts are valid and error if not.
   TEZ-2734. Add a test to verify the filename generated by OnDiskMerge.
   TEZ-2732. DefaultSorter throws ArrayIndex exceptions on 2047 Mb size sort buffers
   TEZ-2687. ATS History shutdown happens before the min-held containers are released

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index e3e9e74..312ddcd 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -25,6 +25,7 @@ import java.util.Map;
 
 import javax.annotation.Nullable;
 
+import org.apache.tez.common.JavaOptsChecker;
 import org.apache.tez.common.RPCUtil;
 import org.apache.tez.common.counters.Limits;
 import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
@@ -122,6 +123,7 @@ public class TezClient {
   @VisibleForTesting
   final ServicePluginsDescriptor servicePluginsDescriptor;
   private HistoryACLPolicyManager historyACLPolicyManager;
+  private JavaOptsChecker javaOptsChecker = null;
 
   private int preWarmDAGCounter = 0;
 
@@ -365,6 +367,28 @@ public class TezClient {
       }
     }
 
+    if (this.amConfig.getTezConfiguration().getBoolean(
+        TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED,
+        TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT)) {
+      String javaOptsCheckerClassName = this.amConfig.getTezConfiguration().get(
+          TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS, "");
+      if (!javaOptsCheckerClassName.isEmpty()) {
+        try {
+          javaOptsChecker = ReflectionUtils.createClazzInstance(javaOptsCheckerClassName);
+        } catch (Exception e) {
+          LOG.warn("Failed to initialize configured Java Opts Checker"
+              + " (" + TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS
+              + ") , checkerClass=" + javaOptsCheckerClassName
+              + ". Disabling checker.", e);
+          javaOptsChecker = null;
+        }
+      } else {
+        javaOptsChecker = new JavaOptsChecker();
+      }
+
+    }
+
+
     if (isSession) {
       LOG.info("Session mode. Starting session.");
       TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
@@ -390,7 +414,7 @@ public class TezClient {
                 sessionAppId,
                 null, clientName, amConfig,
                 tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo,
-                historyACLPolicyManager, servicePluginsDescriptor);
+                historyACLPolicyManager, servicePluginsDescriptor, javaOptsChecker);
   
         // Set Tez Sessions to not retry on AM crashes if recovery is disabled
         if (!amConfig.getTezConfiguration().getBoolean(
@@ -473,7 +497,8 @@ public class TezClient {
 
     Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
     DAGPlan dagPlan = TezClientUtils.prepareAndCreateDAGPlan(dag, amConfig, tezJarResources,
-        usingTezArchiveDeploy, sessionCredentials, aclConfigs, servicePluginsDescriptor);
+        usingTezArchiveDeploy, sessionCredentials, aclConfigs, servicePluginsDescriptor,
+        javaOptsChecker);
 
     SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder();
     requestBuilder.setDAGPlan(dagPlan).build();
@@ -807,7 +832,7 @@ public class TezClient {
           .createApplicationSubmissionContext( 
               appId, dag, dag.getName(), amConfig, tezJarResources, credentials,
               usingTezArchiveDeploy, apiVersionInfo, historyACLPolicyManager,
-              servicePluginsDescriptor);
+              servicePluginsDescriptor, javaOptsChecker);
       LOG.info("Submitting DAG to YARN"
           + ", applicationId=" + appId
           + ", dagName=" + dag.getName());

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index ecf5c07..8f1eb7f 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -39,6 +39,7 @@ import java.util.Map.Entry;
 
 import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
+import org.apache.tez.common.JavaOptsChecker;
 import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
 import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
 import org.slf4j.Logger;
@@ -419,7 +420,7 @@ public class TezClientUtils {
       AMConfiguration amConfig, Map<String, LocalResource> tezJarResources,
       Credentials sessionCreds, boolean tezLrsAsArchive,
       TezApiVersionInfo apiVersionInfo, HistoryACLPolicyManager historyACLPolicyManager,
-      ServicePluginsDescriptor servicePluginsDescriptor)
+      ServicePluginsDescriptor servicePluginsDescriptor, JavaOptsChecker javaOptsChecker)
       throws IOException, YarnException {
 
     Preconditions.checkNotNull(sessionCreds);
@@ -609,7 +610,7 @@ public class TezClientUtils {
     if(dag != null) {
       
       DAGPlan dagPB = prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive,
-          sessionCreds, servicePluginsDescriptor);
+          sessionCreds, servicePluginsDescriptor, javaOptsChecker);
 
       // emit protobuf DAG file style
       Path binaryPath = TezCommonUtils.getTezBinPlanStagingPath(tezSysStagingPath);
@@ -685,19 +686,22 @@ public class TezClientUtils {
   
   static DAGPlan prepareAndCreateDAGPlan(DAG dag, AMConfiguration amConfig,
       Map<String, LocalResource> tezJarResources, boolean tezLrsAsArchive,
-      Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor) throws
IOException {
+      Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor,
+      JavaOptsChecker javaOptsChecker) throws IOException {
     return prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive, credentials,
-        null, servicePluginsDescriptor);
+        null, servicePluginsDescriptor, javaOptsChecker);
   }
 
   static DAGPlan prepareAndCreateDAGPlan(DAG dag, AMConfiguration amConfig,
       Map<String, LocalResource> tezJarResources, boolean tezLrsAsArchive,
       Credentials credentials, Map<String, String> additionalDAGConfigs,
-      ServicePluginsDescriptor servicePluginsDescriptor) throws IOException {
+      ServicePluginsDescriptor servicePluginsDescriptor,
+      JavaOptsChecker javaOptsChecker) throws IOException {
     Credentials dagCredentials = setupDAGCredentials(dag, credentials,
         amConfig.getTezConfiguration());
     return dag.createDag(amConfig.getTezConfiguration(), dagCredentials, tezJarResources,
-        amConfig.getBinaryConfLR(), tezLrsAsArchive, additionalDAGConfigs, servicePluginsDescriptor);
+        amConfig.getBinaryConfLR(), tezLrsAsArchive, additionalDAGConfigs, servicePluginsDescriptor,
+        javaOptsChecker);
   }
   
   static void maybeAddDefaultLoggingJavaOpts(String logLevel, List<String> vargs) {
@@ -726,21 +730,39 @@ public class TezClientUtils {
     }
     return StringUtils.join(vargs, " ").trim();
   }
-  
+
+  @Private
+  public static String addDefaultsToTaskLaunchCmdOpts(String vOpts, Configuration conf)
+      throws TezException {
+    return addDefaultsToTaskLaunchCmdOpts(vOpts, conf, null);
+  }
+
   @Private
-  public static String addDefaultsToTaskLaunchCmdOpts(String vOpts, Configuration conf) {
+  public static String addDefaultsToTaskLaunchCmdOpts(String vOpts, Configuration conf,
+      JavaOptsChecker javaOptsChecker) throws TezException {
     String vConfigOpts = "";
     String taskDefaultOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
         TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
     if (taskDefaultOpts != null && !taskDefaultOpts.isEmpty()) {
       vConfigOpts = taskDefaultOpts + " ";
     }
+    String defaultTaskCmdOpts = TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT;
+    if (vOpts != null && !vOpts.isEmpty()) {
+      // Only use defaults if nothing is specified by the user
+      defaultTaskCmdOpts = "";
+    }
+
     vConfigOpts = vConfigOpts + conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS,
-        TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT);
+        defaultTaskCmdOpts);
     if (vConfigOpts != null && !vConfigOpts.isEmpty()) {
       // Add options specified in the DAG at the end.
       vOpts = vConfigOpts + " " + vOpts;
     }
+
+    if (javaOptsChecker != null) {
+      javaOptsChecker.checkOpts(vOpts);
+    }
+
     return vOpts;
   }
 
@@ -986,6 +1008,7 @@ public class TezClientUtils {
     amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
         tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
             TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
+
     return amOpts;
   }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java b/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java
new file mode 100644
index 0000000..7e7c231
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.tez.dag.api.TezException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Unstable
+@Private
+public class JavaOptsChecker {
+
+  private static final Logger LOG = LoggerFactory.getLogger(JavaOptsChecker.class);
+  private static final Pattern pattern = Pattern.compile("\\s*(-XX:([\\+|\\-]?)(\\S+))\\s*");
+
+  public void checkOpts(String opts) throws TezException {
+    Set<String> gcOpts = new TreeSet<String>();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Checking JVM GC opts: " + opts);
+    }
+    Matcher matcher = pattern.matcher(opts);
+    while (matcher.find()) {
+      if (matcher.groupCount() != 3) {
+        continue;
+      }
+
+      String opt = matcher.group(3);
+      if (!opt.endsWith("GC")) {
+        continue;
+      }
+
+      int val = ( matcher.group(2).equals("+") ? 1 : -1 );
+      if (gcOpts.contains(opt)) {
+        val += 1;
+      }
+
+      if (val > 0) {
+        gcOpts.add(opt);
+      } else {
+        gcOpts.remove(opt);
+      }
+    }
+
+    if (gcOpts.size() > 1) {
+      // Handle special case for " -XX:+UseParNewGC -XX:+UseConcMarkSweepGC "
+      // which can be specified together.
+      if (gcOpts.size() == 2) {
+        if (gcOpts.contains("UseParNewGC")
+          && gcOpts.contains("UseConcMarkSweepGC")) {
+          return;
+        }
+      }
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Found clashing GC opts"
+            + ", conflicting GC Values=" + gcOpts);
+      }
+      throw new TezException("Invalid/conflicting GC options found,"
+          + " cmdOpts=\"" + opts + "\"");
+    }
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 78bb660..ad656cd 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -33,6 +33,7 @@ import java.util.Stack;
 import org.apache.commons.collections4.BidiMap;
 import org.apache.commons.collections4.bidimap.DualLinkedHashBidiMap;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tez.common.JavaOptsChecker;
 import org.apache.tez.dag.api.Vertex.VertexExecutionContext;
 import org.apache.tez.dag.api.records.DAGProtos;
 import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
@@ -716,7 +717,7 @@ public class DAG {
                            Map<String, LocalResource> tezJarResources, LocalResource
binaryConfig,
                            boolean tezLrsAsArchive) {
     return createDag(tezConf, extraCredentials, tezJarResources, binaryConfig, tezLrsAsArchive,
-        null, null);
+        null, null, null);
   }
 
   // create protobuf message describing DAG
@@ -724,7 +725,7 @@ public class DAG {
   public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCredentials,
       Map<String, LocalResource> tezJarResources, LocalResource binaryConfig,
       boolean tezLrsAsArchive, Map<String, String> additionalConfigs,
-                                        ServicePluginsDescriptor servicePluginsDescriptor)
{
+      ServicePluginsDescriptor servicePluginsDescriptor, JavaOptsChecker javaOptsChecker)
{
     verify(true);
 
     DAGPlan.Builder dagBuilder = DAGPlan.newBuilder();
@@ -873,8 +874,15 @@ public class DAG {
       taskConfigBuilder.setNumTasks(vertexParallelism);
       taskConfigBuilder.setMemoryMb(vertexTaskResource.getMemory());
       taskConfigBuilder.setVirtualCores(vertexTaskResource.getVirtualCores());
-      taskConfigBuilder.setJavaOpts(
-          TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vertex.getTaskLaunchCmdOpts(), tezConf));
+
+      try {
+        taskConfigBuilder.setJavaOpts(
+            TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vertex.getTaskLaunchCmdOpts(),
tezConf,
+                javaOptsChecker));
+      } catch (TezException e) {
+        throw new TezUncheckedException("Invalid TaskLaunchCmdOpts defined for Vertex "
+            + vertex.getName() + " : " + e.getMessage(), e);
+      }
 
       taskConfigBuilder.setTaskModule(vertex.getName());
       if (!vertexLRs.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 3b7378a..bb404ee 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1264,4 +1264,26 @@ public class TezConfiguration extends Configuration {
       TEZ_PREFIX + "client.diagnostics.wait.timeout-ms";
   @Private
   public static final long TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS_DEFAULT = 3*1000;
+
+  /**
+   * String value.
+   * Ability to provide a different implementation to check/verify java opts defined
+   * for vertices/tasks.
+   * Class has to be an instance of JavaOptsChecker
+   */
+  @Private
+  @ConfigurationScope(Scope.CLIENT)
+  public static final String TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS =
+      TEZ_PREFIX + "java.opts.checker.class";
+
+  /**
+   * Boolean value. Default true.
+   * Ability to disable the Java Opts Checker
+   */
+  @Private
+  @ConfigurationScope(Scope.CLIENT)
+  public static final String TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED =
+      TEZ_PREFIX + "java.opts.checker.enabled";
+  public static final boolean TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT = true;
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 66b273a..2c3cb36 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -502,4 +502,26 @@ public class TestTezClient {
         amConf.getTezConfiguration().getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false));
   }
 
+  public static class InvalidChecker {
+    // No-op class
+  }
+
+  @Test(timeout = 5000)
+  public void testInvalidJavaOptsChecker1() throws YarnException, IOException, ServiceException,
+      TezException {
+    TezConfiguration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS, "InvalidClassName");
+    TezClientForTest client = configureAndCreateTezClient(conf);
+    client.start();
+  }
+
+  @Test(timeout = 5000)
+  public void testInvalidJavaOptsChecker2() throws YarnException, IOException, ServiceException,
+      TezException {
+    TezConfiguration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS, InvalidChecker.class.getName());
+    TezClientForTest client = configureAndCreateTezClient(conf);
+    client.start();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index d1033b2..394e4dd 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -66,6 +66,7 @@ import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezException;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
@@ -225,7 +226,7 @@ public class TestTezClientUtils {
         appId, null, "dagname",
         amConf, m,
         credentials, false,
-        new TezApiVersionInfo(), null, null);
+        new TezApiVersionInfo(), null, null, null);
     assertEquals(testpriority, appcontext.getPriority().getPriority());
   }
 
@@ -262,7 +263,7 @@ public class TestTezClientUtils {
     ApplicationSubmissionContext appSubmissionContext =
         TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
             new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
-            mock(HistoryACLPolicyManager.class), null);
+            mock(HistoryACLPolicyManager.class), null, null);
 
     ContainerLaunchContext amClc = appSubmissionContext.getAMContainerSpec();
     Map<String, ByteBuffer> amServiceData = amClc.getServiceData();
@@ -295,7 +296,7 @@ public class TestTezClientUtils {
     ApplicationSubmissionContext appSubmissionContext =
         TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
             new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
-            mock(HistoryACLPolicyManager.class), null);
+            mock(HistoryACLPolicyManager.class), null, null);
 
     List<String> expectedCommands = new LinkedList<String>();
     expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
@@ -335,7 +336,7 @@ public class TestTezClientUtils {
     ApplicationSubmissionContext appSubmissionContext =
         TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
             new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
-            mock(HistoryACLPolicyManager.class), null);
+            mock(HistoryACLPolicyManager.class), null, null);
 
     List<String> expectedCommands = new LinkedList<String>();
     expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
@@ -401,7 +402,7 @@ public class TestTezClientUtils {
   }
 
   @Test(timeout = 5000)
-  public void testTaskCommandOpts() {
+  public void testTaskCommandOpts() throws TezException {
     TezConfiguration tezConf = new TezConfiguration();
     String taskCommandOpts = "-Xmx 200m -Dtest.property";
     tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskCommandOpts);
@@ -691,4 +692,57 @@ public class TestTezClientUtils {
     assertTrue(confProto.getAmPluginDescriptor().getUberEnabled());
   }
 
+  @Test(timeout = 5000)
+  public void testTaskLaunchCmdOptsSetup() throws TezException {
+    Configuration conf = new Configuration(false);
+    String vOpts = "";
+    String opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf);
+
+    Assert.assertEquals(opts,
+        TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " "
+            + TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT + " " + vOpts);
+
+    vOpts = "foo";
+    opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf);
+
+    Assert.assertEquals(opts,
+        TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + "  " + vOpts);
+
+    String taskOpts = "taskOpts";
+    conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskOpts);
+    opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf);
+
+    Assert.assertEquals(opts,
+        TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT
+            + " " + taskOpts + " " + vOpts);
+
+  }
+
+  @Test(timeout = 5000)
+  public void testClusterTaskLaunchCmdOptsSetup() throws TezException {
+    Configuration conf = new Configuration(false);
+    String adminOpts = "adminOpts";
+    conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, adminOpts);
+
+    String vOpts = "";
+    String opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf);
+
+    Assert.assertEquals(opts,
+        adminOpts + " "
+            + TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT + " " + vOpts);
+
+    vOpts = "foo";
+    opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf);
+
+    Assert.assertEquals(opts, adminOpts + "  " + vOpts);
+
+    String taskOpts = "taskOpts";
+    conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskOpts);
+    opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf);
+
+    Assert.assertEquals(opts, adminOpts + " " + taskOpts + " " + vOpts);
+
+  }
+
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java b/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java
new file mode 100644
index 0000000..07eb9b6
--- /dev/null
+++ b/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestJavaOptsChecker {
+
+  private final JavaOptsChecker javaOptsChecker = new JavaOptsChecker();
+
+  @Test(timeout = 5000)
+  public void testBasicChecker() throws TezException {
+    javaOptsChecker.checkOpts(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT);
+  }
+
+  @Test(timeout = 5000)
+  public void testMultipleGC() {
+    // Clashing GC values
+    String opts = "-XX:+UseConcMarkSweepGC -XX:+UseG1GC -XX:+UseParallelGC ";
+    try {
+      javaOptsChecker.checkOpts(opts);
+      Assert.fail("Expected check to fail with opts=" + opts);
+    } catch (TezException e) {
+      Assert.assertTrue(e.getMessage(),
+          e.getMessage().contains("Invalid/conflicting GC options found"));
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testPositiveNegativeOpts() throws TezException {
+    // Multiple positive GC values
+    String opts = "-XX:+UseConcMarkSweepGC -XX:+UseG1GC -XX:+UseParallelGC -XX:-UseG1GC ";
+    try {
+      javaOptsChecker.checkOpts(opts);
+      Assert.fail("Expected check to fail with opts=" + opts);
+    } catch (TezException e) {
+      Assert.assertTrue(e.getMessage(),
+          e.getMessage().contains("Invalid/conflicting GC options found"));
+    }
+
+    // Positive following a negative is still a positive
+    opts = " -XX:-UseG1GC -XX:+UseParallelGC -XX:-UseG1GC  -XX:+UseG1GC";
+    try {
+      javaOptsChecker.checkOpts(opts);
+      Assert.fail("Expected check to fail with opts=" + opts);
+    } catch (TezException e) {
+      Assert.assertTrue(e.getMessage(),
+          e.getMessage().contains("Invalid/conflicting GC options found"));
+    }
+
+    // Order of positive and negative matters
+    opts = " -XX:+UseG1GC -XX:-UseG1GC -XX:+UseParallelGC -XX:-UseG1GC  -XX:+UseG1GC";
+    try {
+      javaOptsChecker.checkOpts(opts);
+      Assert.fail("Expected check to fail with opts=" + opts);
+    } catch (TezException e) {
+      Assert.assertTrue(e.getMessage(),
+          e.getMessage().contains("Invalid/conflicting GC options found"));
+    }
+
+    // Sanity check for good condition
+    opts = " -XX:+UseG1GC -XX:+UseParallelGC -XX:-UseG1GC ";
+    javaOptsChecker.checkOpts(opts);
+
+    // Invalid negative can be ignored
+    opts = " -XX:+UseG1GC -XX:+UseParallelGC -XX:-UseG1GC -XX:-UseConcMarkSweepGC ";
+    javaOptsChecker.checkOpts(opts);
+
+  }
+
+  @Test(timeout = 5000)
+  public void testSpecialCaseNonConflictingGCOptions() throws TezException {
+    String opts = " -XX:+UseParNewGC -XX:+UseConcMarkSweepGC ";
+    javaOptsChecker.checkOpts(opts);
+
+    opts += " -XX:-UseG1GC ";
+    javaOptsChecker.checkOpts(opts);
+
+    opts += " -XX:+UseG1GC ";
+    try {
+      javaOptsChecker.checkOpts(opts);
+      Assert.fail("Expected check to fail with opts=" + opts);
+    } catch (TezException e) {
+      Assert.assertTrue(e.getMessage(),
+          e.getMessage().contains("Invalid/conflicting GC options found"));
+    }
+
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/fd714c29/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index 7edea2f..005c027 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.common.JavaOptsChecker;
 import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
 import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
 import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
@@ -327,7 +328,7 @@ public class TestDAGPlan {
     dag.addVertex(v1);
 
     try {
-      dag.createDag(new TezConfiguration(false), null, null, null, true, null, null);
+      dag.createDag(new TezConfiguration(false), null, null, null, true);
       fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
     } catch (IllegalStateException e) {
       assertTrue(e.getMessage().contains("AM execution"));
@@ -336,7 +337,7 @@ public class TestDAGPlan {
     dag.setExecutionContext(VertexExecutionContext.createExecuteInContainers(true));
 
     try {
-      dag.createDag(new TezConfiguration(false), null, null, null, true, null, null);
+      dag.createDag(new TezConfiguration(false), null, null, null, true);
       fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
     } catch (IllegalStateException e) {
       assertTrue(e.getMessage().contains("container execution"));
@@ -370,13 +371,14 @@ public class TestDAGPlan {
 
     // Should succeed. Default context is containers.
     dag.createDag(new TezConfiguration(false), null, null, null, true, null,
-        servicePluginsDescriptor);
+        servicePluginsDescriptor, null);
 
 
     // Set execute in AM should fail
     v1.setExecutionContext(VertexExecutionContext.createExecuteInAm(true));
     try {
-      dag.createDag(new TezConfiguration(false), null, null, null, true, null, servicePluginsDescriptor);
+      dag.createDag(new TezConfiguration(false), null, null, null, true, null,
+          servicePluginsDescriptor, null);
       fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
     } catch (IllegalStateException e) {
       assertTrue(e.getMessage().contains("AM execution"));
@@ -384,12 +386,14 @@ public class TestDAGPlan {
 
     // Valid context
     v1.setExecutionContext(validExecContext);
-    dag.createDag(new TezConfiguration(false), null, null, null, true, null, servicePluginsDescriptor);
+    dag.createDag(new TezConfiguration(false), null, null, null, true, null,
+        servicePluginsDescriptor, null);
 
     // Invalid task scheduler
     v1.setExecutionContext(invalidExecContext1);
     try {
-      dag.createDag(new TezConfiguration(false), null, null, null, true, null, servicePluginsDescriptor);
+      dag.createDag(new TezConfiguration(false), null, null, null, true, null,
+          servicePluginsDescriptor, null);
       fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
     } catch (IllegalStateException e) {
       assertTrue(e.getMessage().contains("testvertex"));
@@ -400,7 +404,8 @@ public class TestDAGPlan {
     // Invalid ContainerLauncher
     v1.setExecutionContext(invalidExecContext2);
     try {
-      dag.createDag(new TezConfiguration(false), null, null, null, true, null, servicePluginsDescriptor);
+      dag.createDag(new TezConfiguration(false), null, null, null, true, null,
+          servicePluginsDescriptor, null);
       fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
     } catch (IllegalStateException e) {
       assertTrue(e.getMessage().contains("testvertex"));
@@ -411,7 +416,8 @@ public class TestDAGPlan {
     // Invalid task comm
     v1.setExecutionContext(invalidExecContext3);
     try {
-      dag.createDag(new TezConfiguration(false), null, null, null, true, null, servicePluginsDescriptor);
+      dag.createDag(new TezConfiguration(false), null, null, null, true, null,
+          servicePluginsDescriptor, null);
       fail("Expecting dag create to fail due to invalid ServicePluginDescriptor");
     } catch (IllegalStateException e) {
       assertTrue(e.getMessage().contains("testvertex"));
@@ -456,7 +462,8 @@ public class TestDAGPlan {
     dag.addVertex(v1).addVertex(v2).addEdge(edge);
     dag.setExecutionContext(defaultExecutionContext);
 
-    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true, null,
servicePluginsDescriptor);
+    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true, null,
+        servicePluginsDescriptor, null);
 
     assertEquals(2, dagProto.getVertexCount());
     assertEquals(1, dagProto.getEdgeCount());
@@ -481,4 +488,36 @@ public class TestDAGPlan {
     VertexPlan v2Proto = dagProto.getVertex(1);
     assertFalse(v2Proto.hasExecutionContext());
   }
+
+  @Test(timeout = 5000)
+  public void testInvalidJavaOpts() {
+    DAG dag = DAG.create("testDag");
+    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1")
+        .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
+    v1.setTaskLaunchCmdOpts(" -XX:+UseG1GC ");
+
+    dag.addVertex(v1);
+
+    TezConfiguration conf = new TezConfiguration(false);
+    conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, "  -XX:+UseParallelGC ");
+    try {
+      DAGPlan dagProto = dag.createDag(conf, null, null, null, true, null, null,
+          new JavaOptsChecker());
+      fail("Expected dag creation to fail for invalid java opts");
+    } catch (TezUncheckedException e) {
+      Assert.assertTrue(e.getMessage().contains("Invalid/conflicting GC options"));
+    }
+
+    // Should not fail as java opts valid
+    conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, "  -XX:-UseParallelGC ");
+    DAGPlan dagProto1 = dag.createDag(conf, null, null, null, true, null, null,
+        new JavaOptsChecker());
+
+    // Should not fail as no checker enabled
+    conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, "  -XX:+UseParallelGC ");
+    DAGPlan dagProto2 = dag.createDag(conf, null, null, null, true, null, null, null);
+
+  }
+
 }


Mime
View raw message