hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory.
Date Thu, 03 Nov 2016 02:11:30 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk dcc07ad8c -> 7e521c5a4


HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e521c5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e521c5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e521c5a

Branch: refs/heads/trunk
Commit: 7e521c5a49fbcf88285c102051ea2522edc847b9
Parents: dcc07ad
Author: Andrew Wang <wang@apache.org>
Authored: Wed Nov 2 19:11:05 2016 -0700
Committer: Andrew Wang <wang@apache.org>
Committed: Wed Nov 2 19:11:05 2016 -0700

----------------------------------------------------------------------
 .../hadoop/fs/CommonConfigurationKeysPublic.java     |  2 ++
 .../src/main/resources/core-default.xml              |  2 +-
 .../org/apache/hadoop/conf/TestConfigRedactor.java   |  2 ++
 .../java/org/apache/hadoop/lib/server/Server.java    | 15 ++++++---------
 4 files changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index b5b107c..f23dd51 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -770,7 +770,9 @@ public class CommonConfigurationKeysPublic {
   public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
       "hadoop.security.sensitive-config-keys";
   public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
+      "secret$" + "," +
       "password$" + "," +
+      "ssl.keystore.pass$" + "," +
       "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
       "fs.azure\\.account.key.*" + "," +
       "dfs.webhdfs.oauth2.[a-z]+.token" + "," +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1e15b8e..327acfa 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -504,7 +504,7 @@
 
 <property>
   <name>hadoop.security.sensitive-config-keys</name>
-  <value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
+  <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
   <description>A comma-separated list of regular expressions to match against
       configuration keys that should be redacted where appropriate, for
       example, when logging modified properties during a reconfiguration,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
index 81f8f71..eedb9b2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
@@ -47,6 +47,7 @@ public class TestConfigRedactor {
         "dfs.webhdfs.oauth2.refresh.token",
         "ssl.server.keystore.keypassword",
         "ssl.server.keystore.password",
+        "httpfs.ssl.keystore.pass",
         "hadoop.security.sensitive-config-keys"
     );
     for (String key : sensitiveKeys) {
@@ -60,6 +61,7 @@ public class TestConfigRedactor {
         "fs.defaultFS",
         "dfs.replication",
         "ssl.server.keystore.location",
+        "httpfs.config.dir",
         "hadoop.security.credstore.java-keystore-provider.password-file"
     );
     for (String key : normalKeys) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e521c5a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
index 1a0f9ff..82be027 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.lib.server;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.ConfigRedactor;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.lib.util.Check;
 import org.apache.hadoop.lib.util.ConfigurationUtils;
@@ -482,15 +483,13 @@ public class Server {
     }
 
     ConfigurationUtils.injectDefaults(defaultConf, config);
-
+    ConfigRedactor redactor = new ConfigRedactor(config);
     for (String name : System.getProperties().stringPropertyNames()) {
       String value = System.getProperty(name);
       if (name.startsWith(getPrefix() + ".")) {
         config.set(name, value);
-        if (name.endsWith(".password") || name.endsWith(".secret")) {
-          value = "*MASKED*";
-        }
-        log.info("System property sets  {}: {}", name, value);
+        String redacted = redactor.redact(name, value);
+        log.info("System property sets  {}: {}", name, redacted);
       }
     }
 
@@ -499,10 +498,8 @@ public class Server {
     for (Map.Entry<String, String> entry : config) {
       String name = entry.getKey();
       String value = config.get(entry.getKey());
-      if (name.endsWith(".password") || name.endsWith(".secret")) {
-        value = "*MASKED*";
-      }
-      log.debug("  {}: {}", entry.getKey(), value);
+      String redacted = redactor.redact(name, value);
+      log.debug("  {}: {}", entry.getKey(), redacted);
     }
     log.debug("------------------------------------------------------");
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message