hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject hive git commit: HIVE-9013 : Hive set command exposes metastore db password (Binglin Chang, reviewed by Thejas Nair, Sushanth Sowmyan)
Date Mon, 26 Oct 2015 19:13:46 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 d33e90ccb -> eb582ed84


HIVE-9013 : Hive set command exposes metastore db password (Binglin Chang, reviewed by Thejas
Nair, Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eb582ed8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eb582ed8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eb582ed8

Branch: refs/heads/branch-1
Commit: eb582ed84f0afd30e63101aedb751a8b1a2e5150
Parents: d33e90c
Author: Sushanth Sowmyan <khorgath@gmail.com>
Authored: Mon Oct 26 12:13:10 2015 -0700
Committer: Sushanth Sowmyan <khorgath@gmail.com>
Committed: Mon Oct 26 12:13:37 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   | 33 ++++++++++++
 .../apache/hadoop/hive/conf/TestHiveConf.java   | 25 +++++++++
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 12 -----
 .../hadoop/hive/ql/exec/mr/ExecDriver.java      |  2 +-
 .../hadoop/hive/ql/exec/tez/DagUtils.java       |  2 +-
 .../hive/ql/exec/tez/TezSessionState.java       |  2 +-
 .../hadoop/hive/ql/processors/SetProcessor.java | 12 ++++-
 .../hive/ql/processors/TestSetProcessor.java    | 54 ++++++++++++++++++++
 8 files changed, 125 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4724523..33e20aa 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -26,11 +26,13 @@ import java.io.PrintStream;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -77,6 +79,7 @@ public class HiveConf extends Configuration {
   private static final Map<String, ConfVars> vars = new HashMap<String, ConfVars>();
   private static final Map<String, ConfVars> metaConfs = new HashMap<String, ConfVars>();
   private final List<String> restrictList = new ArrayList<String>();
+  private final Set<String> hiddenSet = new HashSet<String>();
 
   private Pattern modWhiteListPattern = null;
   private volatile boolean isSparkConfigUpdated = false;
@@ -2028,6 +2031,9 @@ public class HiveConf extends Configuration {
     HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list",
         "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
         "Comma separated list of configuration options which are immutable at runtime"),
+    HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list",
+        METASTOREPWD.varname + "," + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname,
+        "Comma separated list of configuration options which should not be read by normal
user like passwords"),
 
     // If this is set all move tasks at the end of a multi-insert query will only begin once
all
     // outputs are ready
@@ -2501,6 +2507,10 @@ public class HiveConf extends Configuration {
     }
   }
 
+  public boolean isHiddenConfig(String name) {
+    return hiddenSet.contains(name);
+  }
+
   /**
    * check whether spark related property is updated, which includes spark configurations,
    * RSC configurations and yarn configuration in Spark on YARN mode.
@@ -2850,6 +2860,7 @@ public class HiveConf extends Configuration {
 
     // setup list of conf vars that are not allowed to change runtime
     setupRestrictList();
+    setupHiddenSet();
 
   }
 
@@ -3169,6 +3180,28 @@ public class HiveConf extends Configuration {
     }
     restrictList.add(ConfVars.HIVE_IN_TEST.varname);
     restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname);
+    restrictList.add(ConfVars.HIVE_CONF_HIDDEN_LIST.varname);
+  }
+
+  private void setupHiddenSet() {
+    String hiddenListStr = this.getVar(ConfVars.HIVE_CONF_HIDDEN_LIST);
+    hiddenSet.clear();
+    if (hiddenListStr != null) {
+      for (String entry : hiddenListStr.split(",")) {
+        hiddenSet.add(entry.trim());
+      }
+    }
+  }
+
+  /**
+   * Strips hidden config entries from configuration
+   */
+  public void stripHiddenConfigurations(Configuration conf) {
+    for (String name : hiddenSet) {
+      if (conf.get(name) != null) {
+        conf.set(name, "");
+      }
+    }
   }
 
   public static boolean isLoadMetastoreConfig() {

http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
index e9bde21..3b7a525 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.conf;
 
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.util.Shell;
 import org.apache.hive.common.util.HiveTestUtils;
@@ -117,4 +118,28 @@ public class TestHiveConf {
     Assert.assertEquals(TimeUnit.NANOSECONDS, HiveConf.unitFor("ns", null));
     Assert.assertEquals(TimeUnit.NANOSECONDS, HiveConf.unitFor("nsecs", null));
   }
+
+  @Test
+  public void testHiddenConfig() throws Exception {
+    HiveConf conf = new HiveConf();
+    // check password configs are hidden
+    Assert.assertTrue(conf.isHiddenConfig(HiveConf.ConfVars.METASTOREPWD.varname));
+    Assert.assertTrue(conf.isHiddenConfig(
+        HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname));
+    // check change hidden list should fail
+    try {
+      final String name = HiveConf.ConfVars.HIVE_CONF_HIDDEN_LIST.varname;
+      conf.verifyAndSet(name, "");
+      Assert.fail("Setting config property " + name + " should fail");
+    } catch (IllegalArgumentException e) {
+      // the verifyAndSet in this case is expected to fail with the IllegalArgumentException
+    }
+    // check stripHiddenConfigurations
+    Configuration conf2 = new Configuration(conf);
+    conf2.set(HiveConf.ConfVars.METASTOREPWD.varname, "password");
+    conf2.set(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, "password");
+    conf.stripHiddenConfigurations(conf2);
+    Assert.assertEquals("", conf2.get(HiveConf.ConfVars.METASTOREPWD.varname));
+    Assert.assertEquals("", conf2.get(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 973456b..04842ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -3905,16 +3905,4 @@ public final class Utilities {
       (loggingLevel.equalsIgnoreCase("PERFORMANCE") || loggingLevel.equalsIgnoreCase("VERBOSE"));
   }
 
-  /**
-   * Strips Hive password details from configuration
-   */
-  public static void stripHivePasswordDetails(Configuration conf) {
-    // Strip out all Hive related password information from the JobConf
-    if (HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREPWD) != null) {
-      HiveConf.setVar(conf, HiveConf.ConfVars.METASTOREPWD, "");
-    }
-    if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD) != null)
{
-      HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD, "");
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index a2cf712..5f5b9e6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -164,7 +164,7 @@ public class ExecDriver extends Task<MapredWork> implements Serializable,
Hadoop
     if (StringUtils.isNotBlank(addedArchives)) {
       HiveConf.setVar(job, ConfVars.HIVEADDEDARCHIVES, addedArchives);
     }
-    Utilities.stripHivePasswordDetails(job);
+    conf.stripHiddenConfigurations(job);
     this.jobExecHelper = new HadoopJobExecHelper(job, console, this, this);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 19da1c3..7822be9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -1018,7 +1018,7 @@ public class DagUtils {
     // Removing job credential entry/ cannot be set on the tasks
     conf.unset("mapreduce.job.credentials.binary");
 
-    Utilities.stripHivePasswordDetails(conf);
+    hiveConf.stripHiddenConfigurations(conf);
     return conf;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 6ed6421..97fd490 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -213,7 +213,7 @@ public class TezSessionState {
     // generate basic tez config
     final TezConfiguration tezConfig = new TezConfiguration(conf);
     tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
-    Utilities.stripHivePasswordDetails(tezConfig);
+    conf.stripHiddenConfigurations(tezConfig);
 
     if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
       int n = HiveConf.getIntVar(conf, ConfVars.HIVE_PREWARM_NUM_CONTAINERS);

http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index 2520b4c..de1cdb8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -63,6 +63,9 @@ public class SetProcessor implements CommandProcessor {
     for (Object one : p.keySet()) {
       String oneProp = (String) one;
       String oneValue = p.getProperty(oneProp);
+      if (ss.getConf().isHiddenConfig(oneProp)) {
+        continue;
+      }
       sortedMap.put(oneProp, oneValue);
     }
 
@@ -89,7 +92,9 @@ public class SetProcessor implements CommandProcessor {
   private void dumpOption(String s) {
     SessionState ss = SessionState.get();
 
-    if (ss.getConf().get(s) != null) {
+    if (ss.getConf().isHiddenConfig(s)) {
+      ss.out.println(s + " is a hidden config");
+    } else if (ss.getConf().get(s) != null) {
       ss.out.println(s + "=" + ss.getConf().get(s));
     } else if (ss.getHiveVariables().containsKey(s)) {
       ss.out.println(s + "=" + ss.getHiveVariables().get(s));
@@ -219,7 +224,10 @@ public class SetProcessor implements CommandProcessor {
       }
     } else if (varname.indexOf(HIVECONF_PREFIX) == 0) {
       String var = varname.substring(HIVECONF_PREFIX.length());
-      if (ss.getConf().get(var) != null) {
+      if (ss.getConf().isHiddenConfig(var)) {
+        ss.out.println(HIVECONF_PREFIX + var + " is a hidden config");
+        return createProcessorSuccessResponse();
+      } if (ss.getConf().get(var) != null) {
         ss.out.println(HIVECONF_PREFIX + var + "=" + ss.getConf().get(var));
         return createProcessorSuccessResponse();
       } else {

http://git-wip-us.apache.org/repos/asf/hive/blob/eb582ed8/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java
new file mode 100644
index 0000000..bff643a
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.processors;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Test;
+
+public class TestSetProcessor {
+
+  @Test
+  public void testHiddenConfig() throws Exception {
+    HiveConf conf = new HiveConf();
+    SessionState.start(conf);
+    SessionState state = SessionState.get();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    state.out = new PrintStream(baos);
+    SetProcessor processor = new SetProcessor();
+    processor.run("");
+    state.out.flush();
+    String ret = baos.toString();
+    Assert.assertFalse(ret.contains(HiveConf.ConfVars.METASTOREPWD.varname + "="));
+    Assert.assertFalse(ret.contains(HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname
+ "="));
+
+    baos = new ByteArrayOutputStream();
+    state.out = new PrintStream(baos);
+    processor.run(HiveConf.ConfVars.METASTOREPWD.varname);
+    state.out.flush();
+    ret = new String(baos.toByteArray());
+    Assert.assertTrue(baos.toString().contains("hidden"));
+  }
+
+}


Mime
View raw message