accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [1/7] git commit: ACCUMULO-3212 Create a copy from CachedConfiguration instead of a brand new Configuration
Date Mon, 20 Oct 2014 22:03:26 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.6 0e4159c3f -> 5f29c9f65
  refs/heads/master 8bea7aba9 -> 3353a4596


ACCUMULO-3212 Create a copy from CachedConfiguration instead of a brand new Configuration

Quick/dirty test shows that creating a Configuration from an already initialized
Configuration object is much faster than a branch new Configuration. This ensures
that the created Configuration still has the necessary DFS properties and extracting
credential providers from HDFS works as expected with the default filesystem.

Added a test with MiniDFS and added an extra utility method to make things cleaner.
Also introduced the use of JUnit's Assume in the test case to check for the Hadoop
2.6.0 dependency on the classpath.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2e6be536
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2e6be536
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2e6be536

Branch: refs/heads/1.6
Commit: 2e6be5363cace79b2cae7c557698664be535edfc
Parents: 0e4159c
Author: Josh Elser <elserj@apache.org>
Authored: Mon Oct 20 16:49:30 2014 -0400
Committer: Josh Elser <elserj@apache.org>
Committed: Mon Oct 20 16:49:30 2014 -0400

----------------------------------------------------------------------
 core/pom.xml                                    |  5 ++
 .../tokens/CredentialProviderToken.java         |  3 +-
 .../conf/CredentialProviderFactoryShim.java     | 16 +++-
 .../accumulo/core/conf/SiteConfiguration.java   |  3 +-
 .../conf/CredentialProviderFactoryShimTest.java | 91 +++++++++++++-------
 5 files changed, 82 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/2e6be536/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 03f6be2..f52f1ab 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -117,6 +117,11 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>org.easymock</groupId>
       <artifactId>easymock</artifactId>
       <scope>test</scope>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/2e6be536/core/src/main/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderToken.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderToken.java
b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderToken.java
index 99a3dbb..baae411 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/CredentialProviderToken.java
@@ -24,6 +24,7 @@ import java.util.Set;
 import jline.internal.Preconditions;
 
 import org.apache.accumulo.core.conf.CredentialProviderFactoryShim;
+import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.hadoop.conf.Configuration;
 
 /**
@@ -44,7 +45,7 @@ public class CredentialProviderToken extends PasswordToken {
   }
 
   protected void setWithCredentialProviders(String name, String credentialProviders) throws
IOException {
-    final Configuration conf = new Configuration(false);
+    final Configuration conf = new Configuration(CachedConfiguration.getInstance());
     conf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, credentialProviders);
 
     char[] password = CredentialProviderFactoryShim.getValueFromCredentialProvider(conf,
name);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/2e6be536/core/src/main/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShim.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShim.java
b/core/src/main/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShim.java
index 28805fd..33b1848 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShim.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShim.java
@@ -25,6 +25,7 @@ import java.util.List;
 
 import jline.internal.Preconditions;
 
+import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.hadoop.conf.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -311,7 +312,20 @@ public class CredentialProviderFactoryShim {
    */
   public static Configuration getConfiguration(String credentialProviders) {
     Preconditions.checkNotNull(credentialProviders);
-    final Configuration conf = new Configuration(false);
+    return getConfiguration(new Configuration(CachedConfiguration.getInstance()), credentialProviders);
+  }
+  
+  /**
+   * Adds the Credential Provider configuration elements to the provided {@link Configuration}.
+   * 
+   * @param conf
+   *          Existing Hadoop Configuration
+   * @param credentialProviders
+   *          Comma-separated list of CredentialProvider URLs
+   */
+  public static Configuration getConfiguration(Configuration conf, String credentialProviders)
{
+    Preconditions.checkNotNull(conf);
+    Preconditions.checkNotNull(credentialProviders);
     conf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, credentialProviders);
     return conf;
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/2e6be536/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
index 9b65e7d..003514f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.log4j.Logger;
 
@@ -132,7 +133,7 @@ public class SiteConfiguration extends AccumuloConfiguration {
     if (null != credProviderPathsValue) {
       // We have configuration for a CredentialProvider
       // Try to pull the sensitive password from there
-      Configuration conf = new Configuration(false);
+      Configuration conf = new Configuration(CachedConfiguration.getInstance());
       conf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, credProviderPathsValue);
       return conf;
     }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/2e6be536/core/src/test/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShimTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShimTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShimTest.java
index 5016750..a154e6d 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShimTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/CredentialProviderFactoryShimTest.java
@@ -26,16 +26,19 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 /**
- * 
+ *
  */
 public class CredentialProviderFactoryShimTest {
 
-  private static boolean isCredentialProviderAvailable = false;
   private static final String populatedKeyStoreName = "/accumulo.jceks", emptyKeyStoreName
= "/empty.jceks";
   private static File emptyKeyStore, populatedKeyStore;
 
@@ -43,21 +46,19 @@ public class CredentialProviderFactoryShimTest {
   public static void checkCredentialProviderAvailable() {
     try {
       Class.forName(CredentialProviderFactoryShim.HADOOP_CRED_PROVIDER_CLASS_NAME);
-      isCredentialProviderAvailable = true;
     } catch (Exception e) {
-      isCredentialProviderAvailable = false;
+      // If we can't load the credential provider class, don't run the tests
+      Assume.assumeNoException(e);
     }
 
-    if (isCredentialProviderAvailable) {
-      URL populatedKeyStoreUrl = CredentialProviderFactoryShimTest.class.getResource(populatedKeyStoreName),
-          emptyKeyStoreUrl = CredentialProviderFactoryShimTest.class.getResource(emptyKeyStoreName);

+    URL populatedKeyStoreUrl = CredentialProviderFactoryShimTest.class.getResource(populatedKeyStoreName),
emptyKeyStoreUrl = CredentialProviderFactoryShimTest.class
+        .getResource(emptyKeyStoreName);
 
-      Assert.assertNotNull("Could not find " + populatedKeyStoreName, populatedKeyStoreUrl);
-      Assert.assertNotNull("Could not find " + emptyKeyStoreName, emptyKeyStoreUrl);
+    Assert.assertNotNull("Could not find " + populatedKeyStoreName, populatedKeyStoreUrl);
+    Assert.assertNotNull("Could not find " + emptyKeyStoreName, emptyKeyStoreUrl);
 
-      populatedKeyStore = new File(populatedKeyStoreUrl.getFile());
-      emptyKeyStore = new File(emptyKeyStoreUrl.getFile());
-    }
+    populatedKeyStore = new File(populatedKeyStoreUrl.getFile());
+    emptyKeyStore = new File(emptyKeyStoreUrl.getFile());
   }
 
   protected String getKeyStoreUrl(File absoluteFilePath) {
@@ -77,7 +78,7 @@ public class CredentialProviderFactoryShimTest {
   protected void checkCredentialProviders(Configuration conf, Map<String,String> expectation)
throws IOException {
     List<String> keys = CredentialProviderFactoryShim.getKeys(conf);
     Assert.assertNotNull(keys);
-    
+
     Assert.assertEquals(expectation.keySet(), new HashSet<String>(keys));
     for (String expectedKey : keys) {
       char[] value = CredentialProviderFactoryShim.getValueFromCredentialProvider(conf, expectedKey);
@@ -88,10 +89,6 @@ public class CredentialProviderFactoryShimTest {
 
   @Test
   public void testExtractFromProvider() throws IOException {
-    if (!isCredentialProviderAvailable) {
-      return;
-    }
-
     String absPath = getKeyStoreUrl(populatedKeyStore);
     Configuration conf = new Configuration();
     conf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, absPath);
@@ -104,10 +101,6 @@ public class CredentialProviderFactoryShimTest {
 
   @Test
   public void testEmptyKeyStoreParses() throws IOException {
-    if (!isCredentialProviderAvailable) {
-      return;
-    }
-
     String absPath = getKeyStoreUrl(emptyKeyStore);
     Configuration conf = new Configuration();
     conf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, absPath);
@@ -117,10 +110,6 @@ public class CredentialProviderFactoryShimTest {
 
   @Test
   public void testEmptyAndPopulatedKeyStores() throws IOException {
-    if (!isCredentialProviderAvailable) {
-      return;
-    }
-
     String populatedAbsPath = getKeyStoreUrl(populatedKeyStore), emptyAbsPath = getKeyStoreUrl(emptyKeyStore);
     Configuration conf = new Configuration();
     conf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, populatedAbsPath + ","
+ emptyAbsPath);
@@ -133,10 +122,6 @@ public class CredentialProviderFactoryShimTest {
 
   @Test
   public void testNonExistentClassesDoesntFail() throws IOException {
-    if (isCredentialProviderAvailable) {
-      return;
-    }
-
     Configuration conf = new Configuration();
     conf.set(CredentialProviderFactoryShim.CREDENTIAL_PROVIDER_PATH, "jceks://file/foo/bar.jceks");
     List<String> keys = CredentialProviderFactoryShim.getKeys(conf);
@@ -156,10 +141,6 @@ public class CredentialProviderFactoryShimTest {
 
   @Test
   public void createKeystoreProvider() throws Exception {
-    if (!isCredentialProviderAvailable) {
-      return;
-    }
-
     File targetDir = new File(System.getProperty("user.dir") + "/target");
     File keystoreFile = new File(targetDir, "create.jks");
     if (keystoreFile.exists()) {
@@ -176,4 +157,48 @@ public class CredentialProviderFactoryShimTest {
 
     Assert.assertArrayEquals(credential, CredentialProviderFactoryShim.getValueFromCredentialProvider(conf,
alias));
   }
+
+  @Test
+  public void extractFromHdfs() throws Exception {
+    File target = new File(System.getProperty("user.dir"), "target");
+    String prevValue = System.setProperty("test.build.data", new File(target, this.getClass().getName()
+ "_minidfs").toString());
+    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new Configuration());
+    builder.numDataNodes(1);
+    MiniDFSCluster dfsCluster = builder.build();
+    try {
+      if (null != prevValue) {
+        System.setProperty("test.build.data", prevValue);
+      } else {
+        System.clearProperty("test.build.data");
+      }
+
+      // One namenode, One configuration
+      Configuration dfsConfiguration = dfsCluster.getConfiguration(0);
+      Path destPath = new Path("/accumulo.jceks");
+      DistributedFileSystem dfs = dfsCluster.getFileSystem();
+      // Put the populated keystore in hdfs
+      dfs.copyFromLocalFile(new Path(populatedKeyStore.toURI()), destPath);
+
+      Configuration cpConf = CredentialProviderFactoryShim.getConfiguration(dfsConfiguration,
"jceks://hdfs/accumulo.jceks");
+
+      // The values in the keystore
+      Map<String,String> expectations = new HashMap<String,String>();
+      expectations.put("key1", "value1");
+      expectations.put("key2", "value2");
+
+      checkCredentialProviders(cpConf, expectations);
+    } finally {
+      dfsCluster.shutdown();
+    }
+  }
+  
+  @Test
+  public void existingConfigurationReturned() {
+    Configuration conf = new Configuration(false);
+    conf.set("foo", "bar");
+    Configuration conf2 = CredentialProviderFactoryShim.getConfiguration(conf, "jceks:///file/accumulo.jceks");
+    // Same object
+    Assert.assertSame(conf, conf2);
+    Assert.assertEquals("bar", conf.get("foo"));
+  }
 }


Mime
View raw message