Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2A8A111F4D for ; Wed, 20 Aug 2014 01:36:32 +0000 (UTC) Received: (qmail 67568 invoked by uid 500); 20 Aug 2014 01:36:31 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 67533 invoked by uid 500); 20 Aug 2014 01:36:31 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 67524 invoked by uid 99); 20 Aug 2014 01:36:31 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 20 Aug 2014 01:36:31 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 7A6FB8C1F31; Wed, 20 Aug 2014 01:36:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Wed, 20 Aug 2014 01:36:31 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/3] git commit: ACCUMULO-3066 Allow credential providers to be used with ClientConfiguration. Repository: accumulo Updated Branches: refs/heads/1.6.1-SNAPSHOT 37f900b75 -> 64e5c6ac8 refs/heads/master c6ed57ed8 -> fbe6ff36a ACCUMULO-3066 Allow credential providers to be used with ClientConfiguration. When the general.security.credential.provider.paths is set in the ClientConfiguration, any Sensitive properties defined within will used instead of any other defined configuration value. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/64e5c6ac Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/64e5c6ac Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/64e5c6ac Branch: refs/heads/1.6.1-SNAPSHOT Commit: 64e5c6ac8ca31fa970a8df90af1ab4f45fd3dca7 Parents: 37f900b Author: Josh Elser Authored: Tue Aug 19 21:11:57 2014 -0400 Committer: Josh Elser Committed: Tue Aug 19 21:11:57 2014 -0400 ---------------------------------------------------------------------- .../client/impl/ServerConfigurationUtil.java | 63 ++++++++++- .../accumulo/core/conf/SiteConfiguration.java | 12 +- .../impl/ServerConfigurationUtilTest.java | 110 +++++++++++++++++++ .../core/conf/SiteConfigurationTest.java | 2 + 4 files changed, 177 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/64e5c6ac/core/src/main/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtil.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtil.java index 8021f76..507b071 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtil.java @@ -16,18 +16,23 @@ */ package org.apache.accumulo.core.client.impl; +import java.io.IOException; import java.util.Iterator; import java.util.Map; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.CredentialProviderFactoryShim; import org.apache.accumulo.core.conf.Property; import org.apache.commons.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * All client side code that needs a server side configuration object should obtain it from here. */ public class ServerConfigurationUtil { + private static final Logger log = LoggerFactory.getLogger(ServerConfigurationUtil.class); @SuppressWarnings("deprecation") public static AccumuloConfiguration getConfiguration(Instance instance) { return instance.getConfiguration(); @@ -38,15 +43,33 @@ public class ServerConfigurationUtil { return new AccumuloConfiguration() { @Override public String get(Property property) { - if (config.containsKey(property.getKey())) - return config.getString(property.getKey()); + final String key = property.getKey(); + + // Attempt to load sensitive properties from a CredentialProvider, if configured + if (property.isSensitive()) { + org.apache.hadoop.conf.Configuration hadoopConf = getHadoopConfiguration(); + if (null != hadoopConf) { + try { + char[] value = CredentialProviderFactoryShim.getValueFromCredentialProvider(hadoopConf, key); + if (null != value) { + log.trace("Loaded sensitive value for {} from CredentialProvider", key); + return new String(value); + } else { + log.trace("Tried to load sensitive value for {} from CredentialProvider, but none was found", key); + } + } catch (IOException e) { + log.warn("Failed to extract sensitive property ({}) from Hadoop CredentialProvider, falling back to base AccumuloConfiguration", key, e); + } + } + } + if (config.containsKey(key)) + return config.getString(key); else return base.get(property); } @Override public void getProperties(Map props, PropertyFilter filter) { - base.getProperties(props, filter); @SuppressWarnings("unchecked") @@ -56,6 +79,40 @@ public class ServerConfigurationUtil { if (filter.accept(key)) props.put(key, config.getString(key)); } + + // Attempt to load sensitive properties from a CredentialProvider, if configured + org.apache.hadoop.conf.Configuration hadoopConf = getHadoopConfiguration(); + if (null != hadoopConf) { + try { + for (String key : CredentialProviderFactoryShim.getKeys(hadoopConf)) { + if (!Property.isValidPropertyKey(key) || !Property.isSensitive(key)) { + continue; + } + + if (filter.accept(key)) { + char[] value = CredentialProviderFactoryShim.getValueFromCredentialProvider(hadoopConf, key); + if (null != value) { + props.put(key, new String(value)); + } + } + } + } catch (IOException e) { + log.warn("Failed to extract sensitive properties from Hadoop CredentialProvider, falling back to accumulo-site.xml", e); + } + } + } + + private org.apache.hadoop.conf.Configuration getHadoopConfiguration() { + String credProviderPaths = config.getString(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey()); + if (null != credProviderPaths && !credProviderPaths.isEmpty()) { + org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); + hadoopConf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, credProviderPaths); + return hadoopConf; + } + + log.trace("Did not find credential provider configuration in ClientConfiguration"); + + return null; } }; http://git-wip-us.apache.org/repos/asf/accumulo/blob/64e5c6ac/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java index f5cfb2b..12f3ad2 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java @@ -77,9 +77,6 @@ public class SiteConfiguration extends AccumuloConfiguration { try { char[] value = CredentialProviderFactoryShim.getValueFromCredentialProvider(hadoopConf, key); if (null != value) { - if (log.isTraceEnabled()) { - log.trace("Loaded " + key + "=" + new String(value) + " from CredentialProvider"); - } return new String(value); } } catch (IOException e) { @@ -102,6 +99,11 @@ public class SiteConfiguration extends AccumuloConfiguration { public void getProperties(Map props, PropertyFilter filter) { parent.getProperties(props, filter); + for (Entry entry : getXmlConfig()) + if (filter.accept(entry.getKey())) + props.put(entry.getKey(), entry.getValue()); + + // CredentialProvider should take precedence over site Configuration hadoopConf = getHadoopConfiguration(); if (null != hadoopConf) { try { @@ -121,10 +123,6 @@ public class SiteConfiguration extends AccumuloConfiguration { log.warn("Failed to extract sensitive properties from Hadoop CredentialProvider, falling back to accumulo-site.xml", e); } } - - for (Entry entry : getXmlConfig()) - if (filter.accept(entry.getKey())) - props.put(entry.getKey(), entry.getValue()); } protected Configuration getHadoopConfiguration() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/64e5c6ac/core/src/test/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtilTest.java b/core/src/test/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtilTest.java new file mode 100644 index 0000000..1124283 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/client/impl/ServerConfigurationUtilTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.impl; + +import java.io.File; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.AccumuloConfiguration.AllFilter; +import org.apache.accumulo.core.conf.CredentialProviderFactoryShim; +import org.apache.accumulo.core.conf.DefaultConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ServerConfigurationUtilTest { + + private static boolean isCredentialProviderAvailable = false; + private static final String keystoreName = "/site-cfg.jceks"; + + //site-cfg.jceks={'ignored.property'=>'ignored', 'instance.secret'=>'mysecret', 'general.rpc.timeout'=>'timeout'} + private static File keystore; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + try { + Class.forName(CredentialProviderFactoryShim.HADOOP_CRED_PROVIDER_CLASS_NAME); + isCredentialProviderAvailable = true; + } catch (Exception e) { + isCredentialProviderAvailable = false; + } + + if (isCredentialProviderAvailable) { + URL keystoreUrl = ServerConfigurationUtilTest.class.getResource(keystoreName); + + Assert.assertNotNull("Could not find " + keystoreName, keystoreUrl); + + keystore = new File(keystoreUrl.getFile()); + } + } + + protected String getKeyStoreUrl(File absoluteFilePath) { + return "jceks://file" + absoluteFilePath.getAbsolutePath(); + } + + @Test + public void loadSensitivePropertyFromCredentialProvider() { + if (!isCredentialProviderAvailable) { + return; + } + + String absPath = getKeyStoreUrl(keystore); + ClientConfiguration clientConf = new ClientConfiguration(); + clientConf.addProperty(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), absPath); + + AccumuloConfiguration accClientConf = ServerConfigurationUtil.convertClientConfig(DefaultConfiguration.getInstance(), clientConf); + Assert.assertEquals("mysecret", accClientConf.get(Property.INSTANCE_SECRET)); + } + + @Test + public void defaultValueForSensitiveProperty() { + if (!isCredentialProviderAvailable) { + return; + } + + ClientConfiguration clientConf = new ClientConfiguration(); + + AccumuloConfiguration accClientConf = ServerConfigurationUtil.convertClientConfig(DefaultConfiguration.getInstance(), clientConf); + Assert.assertEquals(Property.INSTANCE_SECRET.getDefaultValue(), accClientConf.get(Property.INSTANCE_SECRET)); + } + + @Test + public void sensitivePropertiesIncludedInProperties() { + if (!isCredentialProviderAvailable) { + return; + } + + String absPath = getKeyStoreUrl(keystore); + ClientConfiguration clientConf = new ClientConfiguration(); + clientConf.addProperty(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), absPath); + + AccumuloConfiguration accClientConf = ServerConfigurationUtil.convertClientConfig(DefaultConfiguration.getInstance(), clientConf); + Map props = new HashMap(); + accClientConf.getProperties(props, new AllFilter()); + + // Only sensitive properties are added + Assert.assertEquals(Property.GENERAL_RPC_TIMEOUT.getDefaultValue(), props.get(Property.GENERAL_RPC_TIMEOUT.getKey())); + // Only known properties are added + Assert.assertFalse(props.containsKey("ignored.property")); + Assert.assertEquals("mysecret", props.get(Property.INSTANCE_SECRET.getKey())); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/64e5c6ac/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java index 822e09a..1783a28 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java @@ -50,6 +50,8 @@ public class SiteConfigurationTest { SiteConfiguration siteCfg = EasyMock.createMockBuilder(SiteConfiguration.class).addMockedMethod("getHadoopConfiguration") .withConstructor(AccumuloConfiguration.class).withArgs(DefaultConfiguration.getInstance()).createMock(); + + siteCfg.set(Property.INSTANCE_SECRET, "ignored"); // site-cfg.jceks={'ignored.property'=>'ignored', 'instance.secret'=>'mysecret', 'general.rpc.timeout'=>'timeout'} URL keystore = SiteConfigurationTest.class.getResource("/site-cfg.jceks");