tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-3326. Display JVM system properties in AM and task logs. (Eric Badger via hitesh)
Date Thu, 01 Sep 2016 21:28:17 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 7e15ba5aa -> 55df029a8


TEZ-3326. Display JVM system properties in AM and task logs. (Eric Badger via hitesh)

(cherry picked from commit ae984705e01b357ba5b7fc972b58be4d75f5016d)

Conflicts:
	CHANGES.txt

(cherry picked from commit b6b04de4627f096f6b182cb44185158efdb6d561)

Conflicts:
	tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
	tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/55df029a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/55df029a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/55df029a

Branch: refs/heads/branch-0.7
Commit: 55df029a87004f9ef8a9d4ca223fa016b8353387
Parents: 7e15ba5
Author: Hitesh Shah <hitesh@apache.org>
Authored: Thu Sep 1 14:13:49 2016 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Thu Sep 1 14:27:57 2016 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../java/org/apache/tez/common/TezUtils.java    | 18 ++++++++++++++
 .../apache/tez/dag/api/TezConfiguration.java    | 17 +++++++++++++
 .../org/apache/tez/common/TestTezUtils.java     | 26 ++++++++++++++++++++
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  9 +++++++
 .../org/apache/tez/runtime/task/TezChild.java   | 10 ++++++++
 6 files changed, 81 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1aeed7d..3bba3e4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3326. Display JVM system properties in AM and task logs.
   TEZ-3009. Errors that occur during container task acquisition are not logged.
   TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
   TEZ-3286. Allow clients to set processor reserved memory per vertex (instead of per container).

http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 93d373b..3aa1914 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -20,6 +20,7 @@ package org.apache.tez.common;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.dag.api.records.DAGProtos;
@@ -66,6 +68,22 @@ public class TezUtils {
     TezClientUtils.addLog4jSystemProperties(logLevel, vargs);
   }
 
+  public static String getSystemPropertiesToLog(Configuration conf) {
+    Collection <String> keys = conf.getTrimmedStringCollection(
+        TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG);
+    if (keys.isEmpty()) {
+      keys = TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT;
+    }
+    StringBuilder sb = new StringBuilder();
+    sb.append("\n/************************************************************\n");
+    sb.append("[system properties]\n");
+    for (String key : keys) {
+      sb.append(key).append(": ").append(System.getProperty(key)).append('\n');
+    }
+    sb.append("************************************************************/");
+    return sb.toString();
+  }
+
 
   /**
    * Convert a Configuration to compressed ByteString using Protocol buffer

http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index a96d560..9b32b5f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -19,7 +19,11 @@
 package org.apache.tez.dag.api;
 
 import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -1450,4 +1454,17 @@ public class TezConfiguration extends Configuration {
       TEZ_PREFIX + "java.opts.checker.enabled";
   public static final boolean TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT = true;
 
+  /**
+   * String value. Determines what JVM properties will be logged for debugging purposes
+   * in the AM and Task runtime logs.
+   */
+  @ConfigurationScope(Scope.AM)
+  public static final String TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG  =
+      TEZ_PREFIX + "tez.jvm.system-properties-to-log";
+  public static final List<String> TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT =
+      Collections.unmodifiableList(Arrays.asList(
+          "os.name","os.version","java.home","java.runtime.version",
+          "java.vendor","java.version","java.vm.name","java.class.path",
+          "java.io.tmpdir","user.dir","user.name"));
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index c88fa67..2eab776 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -17,11 +17,16 @@
 
 package org.apache.tez.common;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 import java.util.BitSet;
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.UserPayload;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -225,5 +230,26 @@ public class TestTezUtils {
 
   }
 
+  @Test
+  public void testLogSystemProperties() throws Exception {
+    Configuration conf = new Configuration();
+    // test default logging
+    conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, " ");
+    String value = TezUtils.getSystemPropertiesToLog(conf);
+    for(String key: TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT) {
+      assertTrue(value.contains(key));
+    }
+
+    // test logging of selected keys
+    String classpath = "java.class.path";
+    String os = "os.name";
+    String version = "java.version";
+    conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ", " + os);
+    value = TezUtils.getSystemPropertiesToLog(conf);
+    assertNotNull(value);
+    assertTrue(value.contains(classpath));
+    assertTrue(value.contains(os));
+    assertFalse(value.contains(version));
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 2b33686..673f2fe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -58,6 +58,7 @@ import java.util.regex.Pattern;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.Options;
+import org.apache.tez.common.TezUtils;
 import org.apache.tez.client.CallerContext;
 import org.apache.tez.dag.api.SessionNotRunning;
 import org.apache.tez.dag.app.dag.event.DAGAppMasterEventDagCleanup;
@@ -2178,6 +2179,14 @@ public class DAGAppMaster extends AbstractService {
       ShutdownHookManager.get().addShutdownHook(
         new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
 
+      // log the system properties
+      if (LOG.isInfoEnabled()) {
+        String systemPropsToLog = TezUtils.getSystemPropertiesToLog(conf);
+        if (systemPropsToLog != null) {
+          LOG.info(systemPropsToLog);
+        }
+      }
+
       initAndStartAppMaster(appMaster, conf);
 
     } catch (Throwable t) {

http://git-wip-us.apache.org/repos/asf/tez/blob/55df029a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 4f00998..757b886 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -55,6 +55,7 @@ import org.apache.tez.common.ContainerTask;
 import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezLocalResource;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.TezUtils;
 import org.apache.tez.common.TezUtilsInternal;
 import org.apache.tez.common.counters.Limits;
 import org.apache.tez.common.security.JobTokenIdentifier;
@@ -488,6 +489,15 @@ public class TezChild {
     TezUtilsInternal.addUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()),
defaultConf);
     UserGroupInformation.setConfiguration(defaultConf);
     Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
+
+    // log the system properties
+    if (LOG.isInfoEnabled()) {
+      String systemPropsToLog = TezUtils.getSystemPropertiesToLog(defaultConf);
+      if (systemPropsToLog != null) {
+        LOG.info(systemPropsToLog);
+      }
+    }
+
     TezChild tezChild = newTezChild(defaultConf, host, port, containerIdentifier,
         tokenIdentifier, attemptNumber, localDirs, System.getenv(Environment.PWD.name()),
         System.getenv(), pid, new ExecutionContextImpl(System.getenv(Environment.NM_HOST.name())),


Mime
View raw message