tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-3326. Display JVM system properties in AM and task logs. (Eric Badger via hitesh)
Date Thu, 01 Sep 2016 21:14:20 GMT
Repository: tez
Updated Branches:
  refs/heads/master 8126d2eb3 -> ae984705e


TEZ-3326. Display JVM system properties in AM and task logs. (Eric Badger via hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ae984705
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ae984705
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ae984705

Branch: refs/heads/master
Commit: ae984705e01b357ba5b7fc972b58be4d75f5016d
Parents: 8126d2e
Author: Hitesh Shah <hitesh@apache.org>
Authored: Thu Sep 1 14:13:49 2016 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Thu Sep 1 14:13:49 2016 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +++
 .../java/org/apache/tez/common/TezUtils.java    | 18 ++++++++++++++
 .../apache/tez/dag/api/TezConfiguration.java    | 19 ++++++++++++++
 .../org/apache/tez/common/TestTezUtils.java     | 26 ++++++++++++++++++++
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  8 ++++++
 .../org/apache/tez/runtime/task/TezChild.java   |  9 +++++++
 6 files changed, 83 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index baade62..b73dd3f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3326. Display JVM system properties in AM and task logs.
   TEZ-3009. Errors that occur during container task acquisition are not logged.
   TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher.
   TEZ-3395. Refactor ShuffleVertexManager to make parts of it re-usable in other plugins.
@@ -101,6 +102,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3326. Display JVM system properties in AM and task logs.
   TEZ-3009. Errors that occur during container task acquisition are not logged.
   TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher.
   TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
@@ -586,6 +588,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3326. Display JVM system properties in AM and task logs.
   TEZ-3009. Errors that occur during container task acquisition are not logged.
   TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
   TEZ-3286. Allow clients to set processor reserved memory per vertex (instead of per container).

http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 93d373b..3aa1914 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -20,6 +20,7 @@ package org.apache.tez.common;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.dag.api.records.DAGProtos;
@@ -66,6 +68,22 @@ public class TezUtils {
     TezClientUtils.addLog4jSystemProperties(logLevel, vargs);
   }
 
+  public static String getSystemPropertiesToLog(Configuration conf) {
+    Collection <String> keys = conf.getTrimmedStringCollection(
+        TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG);
+    if (keys.isEmpty()) {
+      keys = TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT;
+    }
+    StringBuilder sb = new StringBuilder();
+    sb.append("\n/************************************************************\n");
+    sb.append("[system properties]\n");
+    for (String key : keys) {
+      sb.append(key).append(": ").append(System.getProperty(key)).append('\n');
+    }
+    sb.append("************************************************************/");
+    return sb.toString();
+  }
+
 
   /**
    * Convert a Configuration to compressed ByteString using Protocol buffer

http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 15e937f..5aa8f7e 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -19,7 +19,11 @@
 package org.apache.tez.dag.api;
 
 import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -1647,4 +1651,19 @@ public class TezConfiguration extends Configuration {
   public static final String TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES =
       TEZ_PREFIX + "am.ats.v15.override.summary-types";
   public static final boolean TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT = true;
+
+  /**
+   * String value. Determines what JVM properties will be logged for debugging purposes
+   * in the AM and Task runtime logs.
+   */
+  @ConfigurationScope(Scope.AM)
+  @ConfigurationProperty
+  public static final String TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG  =
+      TEZ_PREFIX + "tez.jvm.system-properties-to-log";
+  public static final List<String> TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT =
+      Collections.unmodifiableList(Arrays.asList(
+          "os.name","os.version","java.home","java.runtime.version",
+          "java.vendor","java.version","java.vm.name","java.class.path",
+          "java.io.tmpdir","user.dir","user.name"));
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index c88fa67..2eab776 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -17,11 +17,16 @@
 
 package org.apache.tez.common;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 import java.util.BitSet;
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.UserPayload;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -225,5 +230,26 @@ public class TestTezUtils {
 
   }
 
+  @Test
+  public void testLogSystemProperties() throws Exception {
+    Configuration conf = new Configuration();
+    // test default logging
+    conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, " ");
+    String value = TezUtils.getSystemPropertiesToLog(conf);
+    for(String key: TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT) {
+      assertTrue(value.contains(key));
+    }
+
+    // test logging of selected keys
+    String classpath = "java.class.path";
+    String os = "os.name";
+    String version = "java.version";
+    conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ", " + os);
+    value = TezUtils.getSystemPropertiesToLog(conf);
+    assertNotNull(value);
+    assertTrue(value.contains(classpath));
+    assertTrue(value.contains(os));
+    assertFalse(value.contains(version));
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 7ca7118..cc07fb7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2356,6 +2356,14 @@ public class DAGAppMaster extends AbstractService {
       ShutdownHookManager.get().addShutdownHook(
         new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
 
+      // log the system properties
+      if (LOG.isInfoEnabled()) {
+        String systemPropsToLog = TezUtils.getSystemPropertiesToLog(conf);
+        if (systemPropsToLog != null) {
+          LOG.info(systemPropsToLog);
+        }
+      }
+
       initAndStartAppMaster(appMaster, conf);
 
     } catch (Throwable t) {

http://git-wip-us.apache.org/repos/asf/tez/blob/ae984705/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 3c2d19a..2255ed7 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -54,6 +54,7 @@ import org.apache.tez.common.ContainerTask;
 import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezLocalResource;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.TezUtils;
 import org.apache.tez.common.TezUtilsInternal;
 import org.apache.tez.common.counters.Limits;
 import org.apache.tez.common.security.JobTokenIdentifier;
@@ -502,6 +503,14 @@ public class TezChild {
 
     HadoopShim hadoopShim = new HadoopShimsLoader(defaultConf).getHadoopShim();
 
+    // log the system properties
+    if (LOG.isInfoEnabled()) {
+      String systemPropsToLog = TezUtils.getSystemPropertiesToLog(defaultConf);
+      if (systemPropsToLog != null) {
+        LOG.info(systemPropsToLog);
+      }
+    }
+
     TezChild tezChild = newTezChild(defaultConf, host, port, containerIdentifier,
         tokenIdentifier, attemptNumber, localDirs, System.getenv(Environment.PWD.name()),
         System.getenv(), pid, new ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())),


Mime
View raw message