hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aihu...@apache.org
Subject [2/2] hive git commit: HIVE-16297: Improving hive logging configuration variables (Vihang Karajgaonkar, reviewed by Peter Vary & Aihua Xu)
Date Tue, 04 Apr 2017 23:29:45 GMT
HIVE-16297: Improving hive logging configuration variables (Vihang Karajgaonkar, reviewed by
Peter Vary & Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4e60ea3f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4e60ea3f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4e60ea3f

Branch: refs/heads/master
Commit: 4e60ea3f786ed597b0594c2452dd7d64c44645b9
Parents: c7a44eb
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Tue Apr 4 19:14:28 2017 -0400
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Tue Apr 4 19:28:46 2017 -0400

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/conf/HiveConfUtil.java | 13 ++++++++++++-
 common/src/java/org/apache/hive/http/ConfServlet.java  | 10 +++++++---
 .../apache/hadoop/hive/ql/exec/FileSinkOperator.java   | 10 ++++------
 .../hive/ql/exec/spark/RemoteHiveSparkClient.java      |  9 +++------
 4 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
index 9ba08e5..dc02803 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
@@ -94,11 +94,22 @@ public class HiveConfUtil {
   public static void stripConfigurations(Configuration conf, Set<String> hiddenSet)
{
     for (String name : hiddenSet) {
       if (conf.get(name) != null) {
-        conf.set(name, "");
+        conf.set(name, StringUtils.EMPTY);
       }
     }
   }
 
+  /**
+   * Searches the given configuration object and replaces all the configuration values for
keys
+   * defined hive.conf.hidden.list by empty String
+   *
+   * @param conf - Configuration object which needs to be modified to remove sensitive keys
+   */
+  public static void stripConfigurations(Configuration conf) {
+    Set<String> hiddenSet = getHiddenSet(conf);
+    stripConfigurations(conf, hiddenSet);
+  }
+
   public static void dumpConfig(Configuration originalConf, StringBuilder sb) {
     Set<String> hiddenSet = getHiddenSet(originalConf);
     sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n");

http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/common/src/java/org/apache/hive/http/ConfServlet.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/http/ConfServlet.java b/common/src/java/org/apache/hive/http/ConfServlet.java
index 253df4f..856a5d2 100644
--- a/common/src/java/org/apache/hive/http/ConfServlet.java
+++ b/common/src/java/org/apache/hive/http/ConfServlet.java
@@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
 
 /**
  * A servlet to print out the running configuration data.
@@ -81,11 +82,14 @@ public class ConfServlet extends HttpServlet {
    * Guts of the servlet - extracted for easy testing.
    */
   static void writeResponse(Configuration conf, Writer out, String format)
-    throws IOException, BadFormatException {
+      throws IOException, BadFormatException {
+    //redact the sensitive information from the configuration values
+    Configuration hconf = new Configuration(conf);
+    HiveConfUtil.stripConfigurations(hconf);
     if (FORMAT_JSON.equals(format)) {
-      Configuration.dumpConfiguration(conf, out);
+      Configuration.dumpConfiguration(hconf, out);
     } else if (FORMAT_XML.equals(format)) {
-      conf.writeXml(out);
+      hconf.writeXml(out);
     } else {
       throw new BadFormatException("Bad format: " + format);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index a9d03d0..4d727ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.CompilationOpContext;
 import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -463,13 +464,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
   }
 
   private void logOutputFormatError(Configuration hconf, HiveException ex) {
-    StringWriter errorWriter = new StringWriter();
+    StringBuilder errorWriter = new StringBuilder();
     errorWriter.append("Failed to create output format; configuration: ");
-    try {
-      Configuration.dumpConfiguration(hconf, errorWriter);
-    } catch (IOException ex2) {
-      errorWriter.append("{ failed to dump configuration: " + ex2.getMessage() + " }");
-    }
+    // redact sensitive information before logging
+    HiveConfUtil.dumpConfig(hconf, errorWriter);
     Properties tdp = null;
     if (this.conf.getTableInfo() != null
         && (tdp = this.conf.getTableInfo().getProperties()) != null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/4e60ea3f/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index 4c69899..4ca8f93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -356,12 +356,9 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
     private void logConfigurations(JobConf localJobConf) {
       if (LOG.isInfoEnabled()) {
         LOG.info("Logging job configuration: ");
-        StringWriter outWriter = new StringWriter();
-        try {
-          Configuration.dumpConfiguration(localJobConf, outWriter);
-        } catch (IOException e) {
-          LOG.warn("Error logging job configuration", e);
-        }
+        StringBuilder outWriter = new StringBuilder();
+        // redact sensitive information before logging
+        HiveConfUtil.dumpConfig(localJobConf, outWriter);
         LOG.info(outWriter.toString());
       }
     }


Mime
View raw message