Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: wang@apache.org
To: common-commits@hadoop.apache.org
Message-Id: <7f799a7da01f431dae1f87e1854da6f0@git.apache.org>
Subject: hadoop git commit: HADOOP-13494. ReconfigurableBase can log sensitive
 information. Contributed by Sean Mackrory.
Date: Tue, 16 Aug 2016 22:17:49 +0000 (UTC)
archived-at: Tue, 16 Aug 2016 22:17:51 -0000

Repository: hadoop
Updated Branches:
  refs/heads/trunk 6c154abd3 -> 4b689e7a7


HADOOP-13494. ReconfigurableBase can log sensitive information. Contributed by Sean Mackrory.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b689e7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b689e7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b689e7a

Branch: refs/heads/trunk
Commit: 4b689e7a758a55cec2ca8398727feefc8ac21bfd
Parents: 6c154ab
Author: Andrew Wang <wang@apache.org>
Authored: Tue Aug 16 15:01:18 2016 -0700
Committer: Andrew Wang <wang@apache.org>
Committed: Tue Aug 16 15:01:18 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/conf/ConfigRedactor.java  | 84 ++++++++++++++++++++
 .../apache/hadoop/conf/ReconfigurableBase.java  | 13 ++-
 .../fs/CommonConfigurationKeysPublic.java       | 14 ++++
 .../src/main/resources/core-default.xml         | 10 +++
 .../apache/hadoop/conf/TestConfigRedactor.java  | 72 +++++++++++++++++
 5 files changed, 190 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
new file mode 100644
index 0000000..0ba756c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
+
+/**
+ * Tool for redacting sensitive information when displaying config parameters.
+ *
+ * <p>Some config parameters contain sensitive information (for example, cloud
+ * storage keys). When these properties are displayed in plaintext, we should
+ * redactor their values as appropriate.
+ */
+public class ConfigRedactor {
+
+  private static final String REDACTED_TEXT = "<redacted>";
+
+  private List<Pattern> compiledPatterns;
+
+  public ConfigRedactor(Configuration conf) {
+    String sensitiveRegexList = conf.get(
+        HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
+        HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT);
+    List<String> sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(","));
+    compiledPatterns = new ArrayList<Pattern>();
+    for (String regex : sensitiveRegexes) {
+      Pattern p = Pattern.compile(regex);
+      compiledPatterns.add(p);
+    }
+  }
+
+  /**
+   * Given a key / value pair, decides whether or not to redact and returns
+   * either the original value or text indicating it has been redacted.
+   *
+   * @param key
+   * @param value
+   * @return Original value, or text indicating it has been redacted
+   */
+  public String redact(String key, String value) {
+    if (configIsSensitive(key)) {
+      return REDACTED_TEXT;
+    }
+    return value;
+  }
+
+  /**
+   * Matches given config key against patterns and determines whether or not
+   * it should be considered sensitive enough to redact in logs and other
+   * plaintext displays.
+   *
+   * @param key
+   * @return True if parameter is considered sensitive
+   */
+  private boolean configIsSensitive(String key) {
+    for (Pattern regex : compiledPatterns) {
+      if (regex.matcher(key).find()) {
+        return true;
+      }
+    }
+    return false;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
index 681ca2b..bdd006d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
@@ -117,17 +117,24 @@ public abstract class ReconfigurableBase
       final Collection<PropertyChange> changes =
           parent.getChangedProperties(newConf, oldConf);
       Map<PropertyChange, Optional<String>> results = Maps.newHashMap();
+      ConfigRedactor oldRedactor = new ConfigRedactor(oldConf);
+      ConfigRedactor newRedactor = new ConfigRedactor(newConf);
       for (PropertyChange change : changes) {
         String errorMessage = null;
+        String oldValRedacted = oldRedactor.redact(change.prop, change.oldVal);
+        String newValRedacted = newRedactor.redact(change.prop, change.newVal);
         if (!parent.isPropertyReconfigurable(change.prop)) {
           LOG.info(String.format(
               "Property %s is not configurable: old value: %s, new value: %s",
-              change.prop, change.oldVal, change.newVal));
+              change.prop,
+              oldValRedacted,
+              newValRedacted));
           continue;
         }
         LOG.info("Change property: " + change.prop + " from \""
-            + ((change.oldVal == null) ? "<default>" : change.oldVal)
-            + "\" to \"" + ((change.newVal == null) ? "<default>" : change.newVal)
+            + ((change.oldVal == null) ? "<default>" : oldValRedacted)
+            + "\" to \""
+            + ((change.newVal == null) ? "<default>" : newValRedacted)
             + "\".");
         try {
           String effectiveValue =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index e746f2b..0a3afb7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -753,5 +753,19 @@ public class CommonConfigurationKeysPublic {
    */
   public static final String  HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY =
       "hadoop.security.credstore.java-keystore-provider.password-file";
+
+  /**
+   * @see
+   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+   * core-default.xml</a>
+   */
+  public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
+      "hadoop.security.sensitive-config-keys";
+  public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
+      "password$" + "," +
+      "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
+      "fs.azure\\.account.key.*" + "," +
+      "dfs.webhdfs.oauth2.[a-z]+.token" + "," +
+      HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f9c3f72..e78795c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -503,6 +503,16 @@
 </property>
 
 <property>
+  <name>hadoop.security.sensitive-config-keys</name>
+  <value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
+  <description>A comma-separated list of regular expressions to match against
+      configuration keys that should be redacted where appropriate, for
+      example, when logging modified properties during a reconfiguration,
+      private credentials should not be logged.
+  </description>
+</property>
+
+<property>
   <name>hadoop.workaround.non.threadsafe.getpwuid</name>
   <value>true</value>
   <description>Some operating systems or authentication modules are known to

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
new file mode 100644
index 0000000..81f8f71
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Tests the tool (and the default expression) for deciding which config
+ * redact.
+ */
+public class TestConfigRedactor {
+  private static final String REDACTED_TEXT = "<redacted>";
+
+  private static final String ORIGINAL_VALUE = "Hello, World!";
+
+  @Test
+  public void redact() throws Exception {
+    Configuration conf = new Configuration();
+    ConfigRedactor redactor = new ConfigRedactor(conf);
+    String processedText;
+
+    List<String> sensitiveKeys = Arrays.asList(
+        "fs.s3a.secret.key",
+        "fs.s3n.awsSecretKey",
+        "fs.azure.account.key.abcdefg.blob.core.windows.net",
+        "dfs.webhdfs.oauth2.access.token",
+        "dfs.webhdfs.oauth2.refresh.token",
+        "ssl.server.keystore.keypassword",
+        "ssl.server.keystore.password",
+        "hadoop.security.sensitive-config-keys"
+    );
+    for (String key : sensitiveKeys) {
+      processedText = redactor.redact(key, ORIGINAL_VALUE);
+      Assert.assertEquals(
+          "Config parameter wasn't redacted and should be: " + key,
+          REDACTED_TEXT, processedText);
+    }
+
+    List<String> normalKeys = Arrays.asList(
+        "fs.defaultFS",
+        "dfs.replication",
+        "ssl.server.keystore.location",
+        "hadoop.security.credstore.java-keystore-provider.password-file"
+    );
+    for (String key : normalKeys) {
+      processedText = redactor.redact(key, ORIGINAL_VALUE);
+      Assert.assertEquals(
+          "Config parameter was redacted and shouldn't be: " + key,
+          ORIGINAL_VALUE, processedText);
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org