tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-1775. Allow setting log level per logger. (sseth)
Date Mon, 22 Dec 2014 21:00:27 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 5972c0623 -> 599bb0fe2


TEZ-1775. Allow setting log level per logger. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/599bb0fe
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/599bb0fe
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/599bb0fe

Branch: refs/heads/branch-0.5
Commit: 599bb0fe2ee6d80125c10eecc2554ddea8f869ef
Parents: 5972c06
Author: Siddharth Seth <sseth@apache.org>
Authored: Mon Dec 22 13:00:21 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Mon Dec 22 13:00:21 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../org/apache/tez/client/TezClientUtils.java   |  47 +++-
 .../apache/tez/dag/api/TezConfiguration.java    |  53 ++++-
 .../org/apache/tez/dag/api/TezConstants.java    |   2 +
 .../apache/tez/client/TestTezClientUtils.java   |  82 ++++++-
 .../apache/tez/common/TezLog4jConfigurator.java |  50 ++++
 .../apache/tez/dag/app/dag/impl/VertexImpl.java |  50 +++-
 .../dag/utils/TaskSpecificLaunchCmdOption.java  |  94 +++++---
 .../resources/tez-container-log4j.properties    |   2 +-
 .../tez/dag/app/dag/impl/TestVertexImpl.java    |   2 +-
 .../tez/dag/app/dag/impl/TestVertexImpl2.java   | 231 +++++++++++++++++++
 .../utils/TestTaskSpecificLaunchCmdOption.java  |  46 +++-
 12 files changed, 607 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d0abe99..8dc8aaa 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
 Release 0.5.4: Unreleased
 
 ALL CHANGES:
+  TEZ-1775. Allow setting log level per logger.
   TEZ-1851. FileSystem counters do not differentiate between different FileSystems.
   TEZ-1852. Get examples to work in LocalMode.
   TEZ-1861. Fix failing test: TestOnFileSortedOutput.

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index de4bdd0..4cac4d2 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -36,6 +36,7 @@ import java.util.TreeMap;
 import java.util.Vector;
 import java.util.Map.Entry;
 
+import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -402,7 +403,9 @@ public class TezClientUtils {
    * @throws IOException
    * @throws YarnException
    */
-  static ApplicationSubmissionContext createApplicationSubmissionContext(
+  @Private
+  @VisibleForTesting
+  public static ApplicationSubmissionContext createApplicationSubmissionContext(
       ApplicationId appId, DAG dag, String amName,
       AMConfiguration amConfig, Map<String, LocalResource> tezJarResources,
       Credentials sessionCreds, boolean tezLrsAsArchive,
@@ -440,7 +443,7 @@ public class TezClientUtils {
     }
 
     // Add Staging dir creds to the list of session credentials.
-    TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] {binaryConfPath}, conf);
+    TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[]{binaryConfPath}, conf);
 
     // Add session specific credentials to the AM credentials.
     amLaunchCredentials.mergeAll(sessionCreds);
@@ -456,11 +459,15 @@ public class TezClientUtils {
     String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability);
     vargs.add(amOpts);
 
-    String amLogLevel = amConfig.getTezConfiguration().get(
+    String amLogLevelString = amConfig.getTezConfiguration().get(
         TezConfiguration.TEZ_AM_LOG_LEVEL,
         TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
+    String[] amLogParams = parseLogParams(amLogLevelString);
+
+    String amLogLevel = amLogParams[0];
     maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs);
 
+
     // FIX sun bug mentioned in TEZ-327
     vargs.add("-Dsun.nio.ch.bugLevel=''");
 
@@ -502,7 +509,8 @@ public class TezClientUtils {
     }
 
     addVersionInfoToEnv(environment, apiVersionInfo);
-    
+    addLogParamsToEnv(environment, amLogParams);
+
     Map<String, LocalResource> amLocalResources =
         new TreeMap<String, LocalResource>();
 
@@ -650,8 +658,9 @@ public class TezClientUtils {
     }
     TezClientUtils.addLog4jSystemProperties(logLevel, vargs);
   }
-  
-  static String maybeAddDefaultLoggingJavaOpts(String logLevel, String javaOpts) {
+
+  @Private
+  public static String maybeAddDefaultLoggingJavaOpts(String logLevel, String javaOpts) {
     List<String> vargs = new ArrayList<String>(5);
     if (javaOpts != null) {
       vargs.add(javaOpts);
@@ -679,10 +688,6 @@ public class TezClientUtils {
       // Add options specified in the DAG at the end.
       vOpts = vConfigOpts + " " + vOpts;
     }
-    
-    vOpts = maybeAddDefaultLoggingJavaOpts(conf.get(
-        TezConfiguration.TEZ_TASK_LOG_LEVEL,
-        TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT), vOpts);
     return vOpts;
   }
 
@@ -690,6 +695,7 @@ public class TezClientUtils {
   @VisibleForTesting
   public static void addLog4jSystemProperties(String logLevel,
       List<String> vargs) {
+    vargs.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
     vargs.add("-Dlog4j.configuration="
         + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
     vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
@@ -920,4 +926,25 @@ public class TezClientUtils {
     }
   }
 
+  @Private
+  public static void addLogParamsToEnv(Map<String, String> environment, String[] logParams)
{
+    if (logParams.length == 2 && !Strings.isNullOrEmpty(logParams[1])) {
+      TezYARNUtils.replaceInEnv(environment, TezConstants.TEZ_CONTAINER_LOG_PARAMS, logParams[1]);
+    }
+  }
+
+  @Private
+  public static String[] parseLogParams(String logConfig) {
+    if (!Strings.isNullOrEmpty(logConfig)) {
+      int separatorIndex = logConfig.indexOf(TezConstants.TEZ_CONTAINER_LOG_PARAMS_SEPARATOR);
+      if (separatorIndex == -1) {
+        return new String[]{logConfig.trim()};
+      } else {
+        return new String[]{logConfig.substring(0, separatorIndex),
+            logConfig.substring(separatorIndex + 1, logConfig.length()).trim()};
+      }
+    } else {
+      return null;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index fd92b1f..e4ca689 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -117,11 +117,41 @@ public class TezConfiguration extends Configuration {
   public static final String TEZ_AM_SESSION_MODE = TEZ_AM_PREFIX + "mode.session";
   public static boolean TEZ_AM_SESSION_MODE_DEFAULT = false;
 
-  /** Root Logging level passed to the Tez app master.*/
+  /**
+   * Root Logging level passed to the Tez app master.
+   *
+   * Simple configuration: Set the log level for all loggers.
+   *   e.g. INFO
+   *   This sets the log level to INFO for all loggers.
+   *
+   * Advanced configuration: Set the log level for all classes, along with a different level
for some.
+   *   e.g. DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO
+   *   This sets the log level for all loggers to DEBUG, expect for the
+   *   org.apache.hadoop.ipc and org.apache.hadoop.security, which are set to INFO
+   *
+   * Note: The global log level must always be the first parameter.
+   *   DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is valid
+   *   org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
+   * */
   public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX + "log.level";
   public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO";
 
-  /** Root Logging level passed to the Tez tasks.*/
+  /**
+   * Root Logging level passed to the Tez tasks.
+   *
+   * Simple configuration: Set the log level for all loggers.
+   *   e.g. INFO
+   *   This sets the log level to INFO for all loggers.
+   *
+   * Advanced configuration: Set the log level for all classes, along with a different level
for some.
+   *   e.g. DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO
+   *   This sets the log level for all loggers to DEBUG, expect for the
+   *   org.apache.hadoop.ipc and org.apache.hadoop.security, which are set to INFO
+   *
+   * Note: The global log level must always be the first parameter.
+   *   DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is valid
+   *   org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
+   * */
   public static final String TEZ_TASK_LOG_LEVEL = TEZ_TASK_PREFIX + "log.level";
   public static final String TEZ_TASK_LOG_LEVEL_DEFAULT = "INFO";
 
@@ -726,6 +756,25 @@ public class TezConfiguration extends Configuration {
       ".launch.cmd-opts";
 
   /**
+   * Task specific log level.
+   *
+   * Simple configuration: Set the log level for all loggers.
+   *   e.g. INFO
+   *   This sets the log level to INFO for all loggers.
+   *
+   * Advanced configuration: Set the log level for all classes, along with a different level
for some.
+   *   e.g. DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO
+   *   This sets the log level for all loggers to DEBUG, expect for the
+   *   org.apache.hadoop.ipc and org.apache.hadoop.security, which are set to INFO
+   *
+   * Note: The global log level must always be the first parameter.
+   *   DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is valid
+   *   org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
+   * */
+  @Unstable
+  public static final String TEZ_TASK_SPECIFIC_LOG_LEVEL = TEZ_PREFIX + "task-specific" +
".log.level";
+
+  /**
    * String value that is a class name.
    * Specify the class to use for logging history data
    */

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index e335da8..bc4208f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -62,6 +62,8 @@ public class TezConstants {
   public static final String TEZ_CONTAINER_LOG_FILE_NAME = "syslog";
   public static final String TEZ_CONTAINER_ERR_FILE_NAME = "stderr";
   public static final String TEZ_CONTAINER_OUT_FILE_NAME = "stdout";
+  public static final String TEZ_CONTAINER_LOG_PARAMS = "TEZ_CONTAINER_LOG_PARAMS";
+  public static final String TEZ_CONTAINER_LOG_PARAMS_SEPARATOR = ";";
 
   public static final String TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME =
     TezConfiguration.TEZ_SESSION_PREFIX + "local-resources.pb";

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index f39c4ea..c8e144f 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -20,6 +20,7 @@ package org.apache.tez.client;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
@@ -29,6 +30,7 @@ import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -41,12 +43,20 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezConstants;
 import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.dag.api.records.DAGProtos;
 import org.junit.Assert;
 import org.junit.Test;
@@ -183,6 +193,75 @@ public class TestTezClientUtils {
     assertFalse(localizedMap.isEmpty());
   }
 
+  @Test(timeout = 5000)
+  public void testAMLoggingOptsSimple() throws IOException, YarnException {
+
+    TezConfiguration tezConf = new TezConfiguration();
+    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN");
+
+    ApplicationId appId = ApplicationId.newInstance(1000, 1);
+    DAG dag = DAG.create("testdag");
+    dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"))
+        .setTaskLaunchCmdOpts("initialLaunchOpts"));
+    AMConfiguration amConf =
+        new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), new Credentials());
+    ApplicationSubmissionContext appSubmissionContext =
+        TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
+            new HashMap<String, LocalResource>(), new Credentials(), false, new TezApiVersionInfo());
+
+    List<String> expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
+    assertEquals(1, commands.size());
+    for (String expectedCmd : expectedCommands) {
+      assertTrue(commands.get(0).contains(expectedCmd));
+    }
+
+    Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
+    String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+    assertNull(logEnv);
+  }
+
+  @Test(timeout = 5000)
+  public void testAMLoggingOptsPerLogger() throws IOException, YarnException {
+
+    TezConfiguration tezConf = new TezConfiguration();
+    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
+
+    ApplicationId appId = ApplicationId.newInstance(1000, 1);
+    DAG dag = DAG.create("testdag");
+    dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"))
+        .setTaskLaunchCmdOpts("initialLaunchOpts"));
+    AMConfiguration amConf =
+        new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), new Credentials());
+    ApplicationSubmissionContext appSubmissionContext =
+        TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
+            new HashMap<String, LocalResource>(), new Credentials(), false, new TezApiVersionInfo());
+
+    List<String> expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
+    assertEquals(1, commands.size());
+    for (String expectedCmd : expectedCommands) {
+      assertTrue(commands.get(0).contains(expectedCmd));
+    }
+
+    Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
+    String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+    assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv);
+  }
 
   @Test(timeout = 5000)
   public void testAMCommandOpts() {
@@ -327,7 +406,8 @@ public class TestTezClientUtils {
     String javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts("FOOBAR", origJavaOpts);
     Assert.assertNotNull(javaOpts);
     Assert.assertTrue(javaOpts.contains("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=FOOBAR")
-        && javaOpts.contains(TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE));
+        && javaOpts.contains(TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE)
+        && javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"));
   }
 
   // To run this test case see TestTezCommonUtils::testLocalResourceVisibility

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java b/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java
new file mode 100644
index 0000000..24731c2
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.util.Properties;
+
+
+import org.apache.log4j.PropertyConfigurator;
+import org.apache.log4j.spi.LoggerRepository;
+import org.apache.tez.dag.api.TezConstants;
+
+public class TezLog4jConfigurator extends PropertyConfigurator {
+
+  public void doConfigure(Properties properties, LoggerRepository repository) {
+    String logParams = System.getenv(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+    if (logParams != null) {
+      String []parts = logParams.split(TezConstants.TEZ_CONTAINER_LOG_PARAMS_SEPARATOR);
+      for (String logParam : parts) {
+        String [] logParamParts = logParam.split("=");
+        if (logParamParts.length == 2) {
+          String loggerName = "log4j.logger." + logParamParts[0];
+          String logLevel = logParamParts[1].toUpperCase();
+          properties.setProperty(loggerName, logLevel);
+        } else {
+          // Cannot use Log4J logging from here.
+          System.out.println("TezLog4jConfigurator Ignoring invalid log parameter [" + logParam
+ "]");
+          continue;
+        }
+      }
+    }
+    super.doConfigure(properties, repository);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 593ecca..9137b1e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -39,6 +39,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import javax.annotation.Nullable;
 
+import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.commons.logging.Log;
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.client.TezClientUtils;
 import org.apache.tez.common.ATSConstants;
 import org.apache.tez.common.ReflectionUtils;
 import org.apache.tez.common.counters.TezCounters;
@@ -68,6 +70,7 @@ import org.apache.tez.dag.api.OutputCommitterDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.RootInputLeafOutput;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.TaskLocationHint;
@@ -691,7 +694,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
   private TaskLocationHint taskLocationHints[];
   private Map<String, LocalResource> localResources;
-  private Map<String, String> environment;
+  private final Map<String, String> environment;
+  private final Map<String, String> environmentTaskSpecific;
+  private final String javaOptsTaskSpecific;
   private final String javaOpts;
   private final ContainerContext containerContext;
   private VertexTerminationCause terminationCause;
@@ -753,9 +758,36 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     this.environment = DagTypeConverters
         .createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig()
             .getEnvironmentSettingList());
-    this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan
-        .getTaskConfig().getJavaOpts() : null;
     this.taskSpecificLaunchCmdOpts = taskSpecificLaunchCmdOption;
+
+    // Set up log properties, including task specific log properties.
+    String javaOptsWithoutLoggerMods =
+        vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan.getTaskConfig().getJavaOpts()
: null;
+    String logString = conf.get(TezConfiguration.TEZ_TASK_LOG_LEVEL, TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT);
+    String [] taskLogParams = TezClientUtils.parseLogParams(logString);
+    this.javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts(taskLogParams[0], javaOptsWithoutLoggerMods);
+
+    if (taskSpecificLaunchCmdOpts.hasModifiedLogProperties()) {
+      String [] taskLogParamsTaskSpecific = taskSpecificLaunchCmdOption.getTaskSpecificLogParams();
+      this.javaOptsTaskSpecific = TezClientUtils
+          .maybeAddDefaultLoggingJavaOpts(taskLogParamsTaskSpecific[0], javaOptsWithoutLoggerMods);
+      if (taskLogParamsTaskSpecific.length == 2 && !Strings.isNullOrEmpty(taskLogParamsTaskSpecific[1]))
{
+        environmentTaskSpecific = new HashMap<String, String>(this.environment.size());
+        environmentTaskSpecific.putAll(environment);
+        TezClientUtils.addLogParamsToEnv(environmentTaskSpecific, taskLogParamsTaskSpecific);
+      } else {
+        environmentTaskSpecific = null;
+      }
+    } else {
+      this.javaOptsTaskSpecific = null;
+      this.environmentTaskSpecific = null;
+    }
+
+    // env for tasks which don't have task-specific configuration. Has to be set up later
to
+    // optionally allow copying this for specific tasks
+    TezClientUtils.addLogParamsToEnv(this.environment, taskLogParams);
+
+
     this.containerContext = new ContainerContext(this.localResources,
         appContext.getCurrentDAG().getCredentials(), this.environment, this.javaOpts, this);
 
@@ -1934,9 +1966,17 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   @VisibleForTesting
   ContainerContext getContainerContext(int taskIdx) {
     if (taskSpecificLaunchCmdOpts.addTaskSpecificLaunchCmdOption(vertexName, taskIdx)) {
-      String jvmOpts = taskSpecificLaunchCmdOpts.getTaskSpecificOption(javaOpts, vertexName,
taskIdx);
+
+      String jvmOpts = javaOptsTaskSpecific != null ? javaOptsTaskSpecific : javaOpts;
+
+      if (taskSpecificLaunchCmdOpts.hasModifiedTaskLaunchOpts()) {
+        jvmOpts = taskSpecificLaunchCmdOpts.getTaskSpecificOption(jvmOpts, vertexName, taskIdx);
+      }
+
       ContainerContext context = new ContainerContext(this.localResources,
-          appContext.getCurrentDAG().getCredentials(), this.environment, jvmOpts);
+          appContext.getCurrentDAG().getCredentials(),
+          this.environmentTaskSpecific != null ? this.environmentTaskSpecific : this.environment,
+          jvmOpts);
       return context;
     } else {
       return this.containerContext;

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
b/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
index a487a3b..59aede8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
@@ -27,6 +27,7 @@ import java.util.regex.Pattern;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.TezClientUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 
 import com.google.common.base.Splitter;
@@ -54,20 +55,22 @@ public class TaskSpecificLaunchCmdOption {
   //Range regex where ':' should always be prepended and appended with digit.
   final static Pattern RANGE_REGEX = Pattern.compile("(\\d+):(\\d+)");
 
-  private Map<String, BitSet> tasksMap;
+  private final Map<String, BitSet> tasksMap;
   //Task specific launch-cmd options
-  private String tsLaunchCmdOpts;
+  private final String tsLaunchCmdOpts;
+  //Task specific log options
+  private final String[] tsLogParams;
 
   public TaskSpecificLaunchCmdOption(Configuration conf) {
-    tsLaunchCmdOpts =
-        conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "");
+    this.tsLaunchCmdOpts =
+        conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS);
+    this.tsLogParams = TezClientUtils
+        .parseLogParams(conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL));
 
-    if (!Strings.isNullOrEmpty(tsLaunchCmdOpts)) {
+    if (shouldParseSpecificTaskList()) {
       this.tasksMap = getSpecificTasks(conf);
-      if (!tasksMap.isEmpty() && tsLaunchCmdOpts.isEmpty()) {
-        LOG.warn(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS
-            + " should be specified for adding task specific launch-cmd options");
-      }
+    } else {
+      this.tasksMap = null;
     }
   }
 
@@ -82,17 +85,40 @@ public class TaskSpecificLaunchCmdOption {
    */
   public String getTaskSpecificOption(String launchCmdOpts, String vertexName,
       int taskIdx) {
-    launchCmdOpts = (launchCmdOpts == null) ? "" : launchCmdOpts;
-    vertexName = vertexName.replaceAll(" ", "");
-    String result =
-        this.tsLaunchCmdOpts.replaceAll("__VERTEX_NAME__", vertexName)
-          .replaceAll("__TASK_INDEX__", Integer.toString(taskIdx));
-    result = (launchCmdOpts + " " + result);
+    if (this.tsLaunchCmdOpts != null) {
+      launchCmdOpts = (launchCmdOpts == null) ? "" : launchCmdOpts;
+      vertexName = vertexName.replaceAll(" ", "");
+      String result =
+          this.tsLaunchCmdOpts.replaceAll("__VERTEX_NAME__", vertexName)
+              .replaceAll("__TASK_INDEX__", Integer.toString(taskIdx));
+      result = (launchCmdOpts + " " + result);
+
+      LOG.info("Launch-cmd options added to vertexName=" + vertexName
+          + ", taskIdx=" + taskIdx + ", tsLaunchCmdOpts=" + result.trim());
+
+      return result.trim();
+    }
+    return launchCmdOpts;
+  }
 
-    LOG.info("Launch-cmd options added to vertexName=" + vertexName
-        + ", taskIdx=" + taskIdx + ", tsLaunchCmdOpts=" + result.trim());
+  public boolean hasModifiedTaskLaunchOpts() {
+    return !Strings.isNullOrEmpty(tsLaunchCmdOpts);
+  }
+
+  /**
+   * Retrieve a parsed form of the log string specified for per-task usage. </p>
+   * The first element of the array is the general log level. </p>
+   * The second level, if it exists, is the additional per logger configuration.
+   *
+   *
+   * @return parsed form of the log string specified. null if none specified
+   */
+  public String[] getTaskSpecificLogParams() {
+    return this.tsLogParams;
+  }
 
-    return result.trim();
+  public boolean hasModifiedLogProperties() {
+    return this.tsLogParams != null;
   }
 
   /**
@@ -111,6 +137,10 @@ public class TaskSpecificLaunchCmdOption {
     return (taskSet == null) ? false : ((taskSet.isEmpty()) ? true : taskSet.get(taskId));
   }
 
+  private boolean shouldParseSpecificTaskList() {
+    return !(Strings.isNullOrEmpty(tsLaunchCmdOpts) && tsLogParams == null);
+  }
+
   /**
    * <pre>
    * Get the set of tasks in the job, which needs task specific launch command arguments
to be
@@ -123,23 +153,25 @@ public class TaskSpecificLaunchCmdOption {
    * </pre>
    *
    * @param conf
-   * @return Map<String, BitSet>
+   * @return a map from the vertex name to a BitSet representing tasks to be instruemented.
null if
+   *         the provided configuration is empty or invalid
    */
   private Map<String, BitSet> getSpecificTasks(Configuration conf) {
     String specificTaskList =
-        conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "");
-    final Map<String, BitSet> resultSet = new HashMap<String, BitSet>();
-    if (specificTaskList.isEmpty() || !isValid(specificTaskList)) {
-      return resultSet; // empty set
-    }
-    Matcher matcher = TASKS_REGEX.matcher(specificTaskList);
-    while (matcher.find()) {
-      String vertexName = matcher.group(1).trim();
-      BitSet taskSet = parseTasks(matcher.group(2).trim());
-      resultSet.put(vertexName, taskSet);
+        conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST);
+    if (!Strings.isNullOrEmpty(specificTaskList) && isValid(specificTaskList)) {
+      final Map<String, BitSet> resultSet = new HashMap<String, BitSet>();
+      Matcher matcher = TASKS_REGEX.matcher(specificTaskList);
+      while (matcher.find()) {
+        String vertexName = matcher.group(1).trim();
+        BitSet taskSet = parseTasks(matcher.group(2).trim());
+        resultSet.put(vertexName, taskSet);
+      }
+      LOG.info("Specific tasks with additional launch-cmd options=" + resultSet);
+      return resultSet;
+    } else {
+      return null;
     }
-    LOG.info("Specific tasks with additional launch-cmd options=" + resultSet);
-    return resultSet;
   }
 
   private boolean isValid(String specificTaskList) {

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/main/resources/tez-container-log4j.properties
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/resources/tez-container-log4j.properties b/tez-dag/src/main/resources/tez-container-log4j.properties
index 8a11f2d..7a2aeab 100644
--- a/tez-dag/src/main/resources/tez-container-log4j.properties
+++ b/tez-dag/src/main/resources/tez-container-log4j.properties
@@ -12,7 +12,7 @@
 # limitations under the License. See accompanying LICENSE file.
 #
 # Define some default values that can be overridden by system properties
-hadoop.root.logger=DEBUG,CLA
+tez.root.logger=INFO,CLA
 
 # Define the root logger to the system property "tez.root.logger".
 log4j.rootLogger=${tez.root.logger}, EventCounter

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 9500c97..3424e70 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -2271,7 +2271,7 @@ public class TestVertexImpl {
     VertexImpl v3 = vertices.get("vertex3");
 
     Assert.assertEquals("x3.y3", v3.getProcessorName());
-    Assert.assertEquals("foo", v3.getJavaOpts());
+    Assert.assertTrue(v3.getJavaOpts().contains("foo"));
 
     Assert.assertEquals(2, v3.getInputSpecList(0).size());
     Assert.assertEquals(2, v3.getInputVerticesCount());

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
new file mode 100644
index 0000000..80fa86a
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.StateChangeNotifier;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.dag.utils.TaskSpecificLaunchCmdOption;
+import org.junit.Test;
+
+/**
+ * Contains additional tests for VertexImpl. Please avoid adding class parameters.
+ */
+public class TestVertexImpl2 {
+
+  @Test(timeout = 5000)
+  public void testTaskLoggingOptsPerLogger() {
+
+    Configuration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO");
+
+    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+    List<String> expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    for (int i = 0 ; i < testInfo.numTasks ; i++) {
+      ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+      String javaOpts = containerContext.getJavaOpts();
+      assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+      for (String expectedCmd : expectedCommands) {
+        assertTrue(javaOpts.contains(expectedCmd));
+      }
+
+      Map<String, String> env = containerContext.getEnvironment();
+      String val = env.get(testInfo.envKey);
+      assertEquals(testInfo.envVal, val);
+      String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+      assertEquals("org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO", logEnvVal);
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testTaskLoggingOptsSimple() {
+
+    Configuration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG");
+
+    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+    List<String> expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    for (int i = 0 ; i < testInfo.numTasks ; i++) {
+      ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+      String javaOpts = containerContext.getJavaOpts();
+      assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+      for (String expectedCmd : expectedCommands) {
+        assertTrue(javaOpts.contains(expectedCmd));
+      }
+
+      Map<String, String> env = containerContext.getEnvironment();
+      String val = env.get(testInfo.envKey);
+      assertEquals(testInfo.envVal, val);
+      String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+      assertNull(logEnvVal);
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testTaskSpecificLoggingOpts() {
+
+    String vertexName = "testvertex";
+    String customJavaOpts = "-Xmx128m";
+
+    Configuration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "INFO");
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);
+
+    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+    // Expected command opts for regular tasks
+    List<String> expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "INFO" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    for (int i = 3 ; i < testInfo.numTasks ; i++) {
+      ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+      String javaOpts = containerContext.getJavaOpts();
+
+      assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+      for (String expectedCmd : expectedCommands) {
+        assertTrue(javaOpts.contains(expectedCmd));
+      }
+
+      Map<String, String> env = containerContext.getEnvironment();
+      String val = env.get(testInfo.envKey);
+      assertEquals(testInfo.envVal, val);
+      String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+      assertNull(logEnvVal);
+    }
+
+    // Expected command opts for instrumented tasks.
+    expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    for (int i = 0 ; i < 3 ; i++) {
+      ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+      String javaOpts = containerContext.getJavaOpts();
+
+      assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+      for (String expectedCmd : expectedCommands) {
+        assertTrue(javaOpts.contains(expectedCmd));
+      }
+
+      Map<String, String> env = containerContext.getEnvironment();
+      String val = env.get(testInfo.envKey);
+      assertEquals(testInfo.envVal, val);
+      String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+      assertEquals("org.apache.tez=INFO", logEnvVal);
+    }
+  }
+
+  private static class LogTestInfoHolder {
+
+    final AppContext mockAppContext;
+    final DAG mockDag;
+    final VertexImpl vertex;
+    final DAGProtos.VertexPlan vertexPlan;
+
+    final int numTasks = 10;
+    final String initialJavaOpts = "initialJavaOpts";
+    final String envKey = "key1";
+    final String envVal = "val1";
+
+    LogTestInfoHolder(Configuration conf) {
+      this(conf, "testvertex");
+    }
+
+    LogTestInfoHolder(Configuration conf, String vertexName) {
+      mockAppContext = mock(AppContext.class);
+      mockDag = mock(DAG.class);
+      doReturn(new Credentials()).when(mockDag).getCredentials();
+      doReturn(mockDag).when(mockAppContext).getCurrentDAG();
+
+      vertexPlan = DAGProtos.VertexPlan.newBuilder()
+          .setName(vertexName)
+          .setTaskConfig(DAGProtos.PlanTaskConfiguration.newBuilder()
+              .setJavaOpts(initialJavaOpts)
+              .setNumTasks(numTasks)
+              .setMemoryMb(1024)
+              .setVirtualCores(1)
+              .setTaskModule("taskmodule")
+              .addEnvironmentSetting(DAGProtos.PlanKeyValuePair.newBuilder()
+                  .setKey(envKey)
+                  .setValue(envVal)
+                  .build())
+              .build())
+          .setType(DAGProtos.PlanVertexType.NORMAL).build();
+
+      vertex =
+          new VertexImpl(TezVertexID.fromString("vertex_1418197758681_0001_1_00"), vertexPlan,
+              "testvertex", conf, mock(EventHandler.class), mock(TaskAttemptListener.class),
+              mock(Clock.class), mock(TaskHeartbeatHandler.class), false, mockAppContext,
+              VertexLocationHint.create(new LinkedList<TaskLocationHint>()), null,
+              new TaskSpecificLaunchCmdOption(conf), mock(StateChangeNotifier.class));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
index aebad74..f22eed6 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.dag.utils;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -40,7 +41,8 @@ public class TestTaskSpecificLaunchCmdOption {
     return new TaskSpecificLaunchCmdOption(conf);
   }
 
-  @Test
+
+  @Test(timeout = 5000)
   public void testTaskSpecificJavaOptions() {
     Random rnd = new Random();
     Configuration conf = new Configuration();
@@ -238,7 +240,7 @@ public class TestTaskSpecificLaunchCmdOption {
         rnd.nextInt(Integer.MAX_VALUE)));
   }
 
-  @Test
+  @Test(timeout = 5000)
   public void testConfigOptions() {
     Configuration conf = new Configuration();
     TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption = getOptions(conf, "", "");
@@ -263,4 +265,44 @@ public class TestTaskSpecificLaunchCmdOption {
     optionStr = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v", 4);
     assertTrue(optionStr.equals("dir=/tmp/v/4"));
   }
+
+
+  @Test(timeout = 5000)
+  public void testTaskSpecificLogOptions() {
+    Configuration conf = new Configuration(false);
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[0,2,5]");
+    TaskSpecificLaunchCmdOption options;
+
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
+    options = new TaskSpecificLaunchCmdOption(conf);
+    assertTrue(options.hasModifiedLogProperties());
+    assertFalse(options.hasModifiedTaskLaunchOpts());
+    assertEquals(2, options.getTaskSpecificLogParams().length);
+
+    conf.unset(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL);
+    options = new TaskSpecificLaunchCmdOption(conf);
+    assertFalse(options.hasModifiedLogProperties());
+    assertFalse(options.hasModifiedTaskLaunchOpts());
+
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG");
+    options = new TaskSpecificLaunchCmdOption(conf);
+    assertTrue(options.hasModifiedLogProperties());
+    assertFalse(options.hasModifiedTaskLaunchOpts());
+    assertEquals(1, options.getTaskSpecificLogParams().length);
+  }
+
+  @Test (timeout=5000)
+  public void testTaskSpecificLogOptionsWithCommandOptions() {
+    Configuration conf = new Configuration(false);
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[0,2,5]");
+    TaskSpecificLaunchCmdOption options;
+
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "-Xmx128m");
+    options = new TaskSpecificLaunchCmdOption(conf);
+    assertTrue(options.hasModifiedLogProperties());
+    assertTrue(options.hasModifiedTaskLaunchOpts());
+    String optionStr = options.getTaskSpecificOption("", "v", 0);
+    assertTrue(optionStr.equals("-Xmx128m"));
+  }
 }


Mime
View raw message