Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6AA65200C69 for ; Fri, 21 Apr 2017 23:35:53 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 695A6160BB2; Fri, 21 Apr 2017 21:35:53 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8A060160BAF for ; Fri, 21 Apr 2017 23:35:52 +0200 (CEST) Received: (qmail 51167 invoked by uid 500); 21 Apr 2017 21:35:46 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 50343 invoked by uid 99); 21 Apr 2017 21:35:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Apr 2017 21:35:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9374BF4A03; Fri, 21 Apr 2017 21:35:45 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: virajith@apache.org To: common-commits@hadoop.apache.org Date: Fri, 21 Apr 2017 21:36:04 -0000 Message-Id: In-Reply-To: <3b0020ec90c64ce0bd631ea43b3e8927@git.apache.org> References: <3b0020ec90c64ce0bd631ea43b3e8927@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/37] hadoop git commit: HADOOP-14241. Add ADLS sensitive config keys to default list. Contributed by John Zhuge. archived-at: Fri, 21 Apr 2017 21:35:53 -0000 HADOOP-14241. Add ADLS sensitive config keys to default list. Contributed by John Zhuge. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0344bea3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0344bea3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0344bea3 Branch: refs/heads/HDFS-9806 Commit: 0344bea3fd4031622edd828a610c9fdc23c53d26 Parents: dd43b89 Author: John Zhuge Authored: Sat Mar 25 11:59:53 2017 -0700 Committer: John Zhuge Committed: Wed Apr 19 11:50:54 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/ConfigRedactor.java | 3 ++- .../hadoop/fs/CommonConfigurationKeysPublic.java | 16 +++++++++------- .../src/main/resources/core-default.xml | 15 ++++++++++++--- .../org/apache/hadoop/conf/TestConfigRedactor.java | 17 ++++++++++++++++- 4 files changed, 39 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java index 0ba756c..90b260e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java @@ -42,7 +42,8 @@ public class ConfigRedactor { String sensitiveRegexList = conf.get( HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS, HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT); - List sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(",")); + List sensitiveRegexes = + Arrays.asList(sensitiveRegexList.trim().split("[,\\s]+")); compiledPatterns = new ArrayList(); for (String regex : sensitiveRegexes) { Pattern p = Pattern.compile(regex); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 6b267dc..5f34b7b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -810,12 +810,14 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = "hadoop.security.sensitive-config-keys"; public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = - "secret$" + "," + - "password$" + "," + - "ssl.keystore.pass$" + "," + - "fs.s3.*[Ss]ecret.?[Kk]ey" + "," + - "fs.azure\\.account.key.*" + "," + - "dfs.webhdfs.oauth2.[a-z]+.token" + "," + - HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS; + String.join(",", + "secret$", + "password$", + "ssl.keystore.pass$", + "fs.s3.*[Ss]ecret.?[Kk]ey", + "fs.azure\\.account.key.*", + "credential$", + "oauth.*token$", + HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 4f37c65..283588a 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -517,9 +517,18 @@ hadoop.security.sensitive-config-keys - secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys - A comma-separated list of regular expressions to match against - configuration keys that should be redacted where appropriate, for + + secret$ + password$ + ssl.keystore.pass$ + fs.s3.*[Ss]ecret.?[Kk]ey + fs.azure.account.key.* + credential$ + oauth.*token$ + hadoop.security.sensitive-config-keys + + A comma-separated or multi-line list of regular expressions to + match configuration keys that should be redacted where appropriate, for example, when logging modified properties during a reconfiguration, private credentials should not be logged. http://git-wip-us.apache.org/repos/asf/hadoop/blob/0344bea3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java index eedb9b2..ba08de4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java @@ -34,15 +34,30 @@ public class TestConfigRedactor { private static final String ORIGINAL_VALUE = "Hello, World!"; @Test - public void redact() throws Exception { + public void testRedactWithCoreDefault() throws Exception { Configuration conf = new Configuration(); + testRedact(conf); + } + + @Test + public void testRedactNoCoreDefault() throws Exception { + Configuration conf = new Configuration(false); + testRedact(conf); + } + + private void testRedact(Configuration conf) throws Exception { ConfigRedactor redactor = new ConfigRedactor(conf); String processedText; List sensitiveKeys = Arrays.asList( "fs.s3a.secret.key", + "fs.s3a.bucket.BUCKET.secret.key", "fs.s3n.awsSecretKey", "fs.azure.account.key.abcdefg.blob.core.windows.net", + "fs.adl.oauth2.refresh.token", + "fs.adl.oauth2.credential", + "dfs.adls.oauth2.refresh.token", + "dfs.adls.oauth2.credential", "dfs.webhdfs.oauth2.access.token", "dfs.webhdfs.oauth2.refresh.token", "ssl.server.keystore.keypassword", --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org