hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jzh...@apache.org
Subject hadoop git commit: HADOOP-14241. Add ADLS sensitive config keys to default list. Contributed by John Zhuge.
Date Wed, 19 Apr 2017 19:07:46 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 e278eb3ab -> 3a39dd539


HADOOP-14241. Add ADLS sensitive config keys to default list. Contributed by John Zhuge.

(cherry picked from commit 0344bea3fd4031622edd828a610c9fdc23c53d26)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a39dd53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a39dd53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a39dd53

Branch: refs/heads/branch-2
Commit: 3a39dd5391527623a70296410f4754d8d235310a
Parents: e278eb3
Author: John Zhuge <jzhuge@apache.org>
Authored: Wed Apr 19 12:04:03 2017 -0700
Committer: John Zhuge <jzhuge@apache.org>
Committed: Wed Apr 19 12:04:03 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/conf/ConfigRedactor.java     |  3 ++-
 .../hadoop/fs/CommonConfigurationKeysPublic.java   | 16 +++++++++-------
 .../src/main/resources/core-default.xml            | 15 ++++++++++++---
 .../org/apache/hadoop/conf/TestConfigRedactor.java | 17 ++++++++++++++++-
 4 files changed, 39 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a39dd53/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
index 0ba756c..90b260e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
@@ -42,7 +42,8 @@ public class ConfigRedactor {
     String sensitiveRegexList = conf.get(
         HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
         HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT);
-    List<String> sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(","));
+    List<String> sensitiveRegexes =
+        Arrays.asList(sensitiveRegexList.trim().split("[,\\s]+"));
     compiledPatterns = new ArrayList<Pattern>();
     for (String regex : sensitiveRegexes) {
       Pattern p = Pattern.compile(regex);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a39dd53/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 86feee2..e771634 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -829,12 +829,14 @@ public class CommonConfigurationKeysPublic {
   public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
       "hadoop.security.sensitive-config-keys";
   public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
-      "secret$" + "," +
-      "password$" + "," +
-      "ssl.keystore.pass$" + "," +
-      "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
-      "fs.azure\\.account.key.*" + "," +
-      "dfs.webhdfs.oauth2.[a-z]+.token" + "," +
-      HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
+      String.join(",",
+          "secret$",
+          "password$",
+          "ssl.keystore.pass$",
+          "fs.s3.*[Ss]ecret.?[Kk]ey",
+          "fs.azure\\.account.key.*",
+          "credential$",
+          "oauth.*token$",
+          HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS);
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a39dd53/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 61c9df7..685d565 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -526,9 +526,18 @@
 
 <property>
   <name>hadoop.security.sensitive-config-keys</name>
-  <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
-  <description>A comma-separated list of regular expressions to match against
-      configuration keys that should be redacted where appropriate, for
+  <value>
+      secret$
+      password$
+      ssl.keystore.pass$
+      fs.s3.*[Ss]ecret.?[Kk]ey
+      fs.azure.account.key.*
+      credential$
+      oauth.*token$
+      hadoop.security.sensitive-config-keys
+  </value>
+  <description>A comma-separated or multi-line list of regular expressions to
+      match configuration keys that should be redacted where appropriate, for
       example, when logging modified properties during a reconfiguration,
       private credentials should not be logged.
   </description>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a39dd53/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
index eedb9b2..ba08de4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
@@ -34,15 +34,30 @@ public class TestConfigRedactor {
   private static final String ORIGINAL_VALUE = "Hello, World!";
 
   @Test
-  public void redact() throws Exception {
+  public void testRedactWithCoreDefault() throws Exception {
     Configuration conf = new Configuration();
+    testRedact(conf);
+  }
+
+  @Test
+  public void testRedactNoCoreDefault() throws Exception {
+    Configuration conf = new Configuration(false);
+    testRedact(conf);
+  }
+
+  private void testRedact(Configuration conf) throws Exception {
     ConfigRedactor redactor = new ConfigRedactor(conf);
     String processedText;
 
     List<String> sensitiveKeys = Arrays.asList(
         "fs.s3a.secret.key",
+        "fs.s3a.bucket.BUCKET.secret.key",
         "fs.s3n.awsSecretKey",
         "fs.azure.account.key.abcdefg.blob.core.windows.net",
+        "fs.adl.oauth2.refresh.token",
+        "fs.adl.oauth2.credential",
+        "dfs.adls.oauth2.refresh.token",
+        "dfs.adls.oauth2.credential",
         "dfs.webhdfs.oauth2.access.token",
         "dfs.webhdfs.oauth2.refresh.token",
         "ssl.server.keystore.keypassword",


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message