tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-1878. Task-specific log level override not working in certain conditions. (sseth) (cherry picked from commit a139989c0b43074514bf936511be4e5111a30344)
Date Mon, 22 Dec 2014 20:31:21 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.6 107a9e625 -> 28d7a0e42


TEZ-1878. Task-specific log level override not working in certain
conditions. (sseth)
(cherry picked from commit a139989c0b43074514bf936511be4e5111a30344)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/28d7a0e4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/28d7a0e4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/28d7a0e4

Branch: refs/heads/branch-0.6
Commit: 28d7a0e4298db3d6dfd23d18616ddd918e864261
Parents: 107a9e6
Author: Siddharth Seth <sseth@apache.org>
Authored: Mon Dec 22 12:30:56 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Mon Dec 22 12:31:19 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/tez/dag/app/dag/impl/VertexImpl.java |  7 +--
 .../tez/dag/app/dag/impl/TestVertexImpl2.java   | 66 ++++++++++++++++++++
 3 files changed, 70 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/28d7a0e4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f32633d..e271a33 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.6.0: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-1878. Task-specific log level override not working in certain conditions.
   TEZ-1886. remove deprecation warnings for tez-ui on the console.
   TEZ-1875. dropdown filters do not work on vertices and task attempts page.
   TEZ-1873. TestTezAMRMClient fails due to host resolution timing out.

http://git-wip-us.apache.org/repos/asf/tez/blob/28d7a0e4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 13993cc..b1c93da 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -779,12 +779,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
       String [] taskLogParamsTaskSpecific = taskSpecificLaunchCmdOption.getTaskSpecificLogParams();
       this.javaOptsTaskSpecific = TezClientUtils
           .maybeAddDefaultLoggingJavaOpts(taskLogParamsTaskSpecific[0], javaOptsWithoutLoggerMods);
+
+      environmentTaskSpecific = new HashMap<String, String>(this.environment.size());
+      environmentTaskSpecific.putAll(environment);
       if (taskLogParamsTaskSpecific.length == 2 && !Strings.isNullOrEmpty(taskLogParamsTaskSpecific[1]))
{
-        environmentTaskSpecific = new HashMap<String, String>(this.environment.size());
-        environmentTaskSpecific.putAll(environment);
         TezClientUtils.addLogParamsToEnv(environmentTaskSpecific, taskLogParamsTaskSpecific);
-      } else {
-        environmentTaskSpecific = null;
       }
     } else {
       this.javaOptsTaskSpecific = null;

http://git-wip-us.apache.org/repos/asf/tez/blob/28d7a0e4/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
index 80fa86a..b4064a0 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
@@ -183,6 +183,72 @@ public class TestVertexImpl2 {
     }
   }
 
+  @Test(timeout = 5000)
+  public void testTaskSpecificLoggingOpts2() {
+
+    String vertexName = "testvertex";
+    String customJavaOpts = "-Xmx128m";
+
+    Configuration conf = new TezConfiguration();
+    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "WARN;org.apache.tez=INFO");
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG");
+    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);
+
+    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+    // Expected command opts for regular tasks
+    List<String> expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    for (int i = 3 ; i < testInfo.numTasks ; i++) {
+      ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+      String javaOpts = containerContext.getJavaOpts();
+
+      assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+      for (String expectedCmd : expectedCommands) {
+        assertTrue(javaOpts.contains(expectedCmd));
+      }
+
+      Map<String, String> env = containerContext.getEnvironment();
+      String val = env.get(testInfo.envKey);
+      assertEquals(testInfo.envVal, val);
+      String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+      assertEquals("org.apache.tez=INFO", logEnvVal);
+    }
+
+    // Expected command opts for instrumented tasks.
+    expectedCommands = new LinkedList<String>();
+    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+        TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+    for (int i = 0 ; i < 3 ; i++) {
+      ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+      String javaOpts = containerContext.getJavaOpts();
+
+      assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+      for (String expectedCmd : expectedCommands) {
+        assertTrue(javaOpts.contains(expectedCmd));
+      }
+
+      Map<String, String> env = containerContext.getEnvironment();
+      String val = env.get(testInfo.envKey);
+      assertEquals(testInfo.envVal, val);
+      String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+      assertNull(logEnvVal);
+    }
+  }
+
+
   private static class LogTestInfoHolder {
 
     final AppContext mockAppContext;


Mime
View raw message