hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject hadoop git commit: HDFS-11687. Add new public encryption APIs required by Hive. (lei)
Date Thu, 04 May 2017 19:07:53 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk c2a52ef9c -> 25f5d9ad5


HDFS-11687. Add new public encryption APIs required by Hive. (lei)

Change-Id: I4a23a00de63ad18022312ceb1f306a87d032d07c


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25f5d9ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25f5d9ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25f5d9ad

Branch: refs/heads/trunk
Commit: 25f5d9ad5ee5ead349d259a99b49541a70b1604d
Parents: c2a52ef
Author: Lei Xu <lei@apache.org>
Authored: Thu May 4 12:06:50 2017 -0700
Committer: Lei Xu <lei@apache.org>
Committed: Thu May 4 12:06:50 2017 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 18 ++---------
 .../hadoop/hdfs/DistributedFileSystem.java      |  9 ++++--
 .../apache/hadoop/hdfs/client/HdfsAdmin.java    | 12 +++++++
 .../apache/hadoop/hdfs/TestEncryptionZones.java |  1 +
 .../org/apache/hadoop/hdfs/TestHdfsAdmin.java   | 33 ++++++++++++++++++++
 5 files changed, 55 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f5d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 187d2e9..3f1f3ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2951,24 +2951,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
 
   /**
    * Probe for encryption enabled on this filesystem.
-   * Note (see HDFS-11689):
-   * Not to throw exception in this method since it would break hive.
-   * Hive accesses this method and assumes no exception would be thrown.
-   * Hive should not access DFSClient since it is InterfaceAudience.Private.
-   * Deprecated annotation is added to trigger build warning at hive side.
-   * Request has been made to Hive to remove access to DFSClient.
    * @return true if encryption is enabled
    */
-  @Deprecated
-  public boolean isHDFSEncryptionEnabled() {
-    boolean result = false;
-    try {
-      result = (getKeyProviderUri() != null);
-    } catch (IOException ioe) {
-      DFSClient.LOG.warn("Exception while checking whether encryption zone "
-            + "is supported, assumes it is not supported", ioe);
-    }
-    return result;
+  boolean isHDFSEncryptionEnabled() throws IOException {
+    return getKeyProviderUri() != null;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f5d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 9e89bc5..7b2f2fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2587,8 +2587,13 @@ public class DistributedFileSystem extends FileSystem {
    */
   @Override
   public Path getTrashRoot(Path path) {
-    if ((path == null) || !dfs.isHDFSEncryptionEnabled()) {
-      return super.getTrashRoot(path);
+    try {
+      if ((path == null) || !dfs.isHDFSEncryptionEnabled()) {
+        return super.getTrashRoot(path);
+      }
+    } catch (IOException ioe) {
+      DFSClient.LOG.warn("Exception while checking whether encryption zone is "
+          + "supported", ioe);
     }
 
     String parentSrc = path.isRoot()?

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f5d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index 456f280..71f6a35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.fs.BlockStoragePolicySpi;
 import org.apache.hadoop.fs.CacheFlag;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -268,6 +269,17 @@ public class HdfsAdmin {
   }
 
   /**
+   * Get KeyProvider if present.
+   *
+   * @return the key provider if encryption is enabled on HDFS.
+   *         Otherwise, it returns null.
+   * @throws IOException on RPC exception to the NN.
+   */
+  public KeyProvider getKeyProvider() throws IOException {
+    return dfs.getClient().getKeyProvider();
+  }
+
+  /**
    * Create an encryption zone rooted at an empty existing directory, using the
    * specified encryption key. An encryption zone has an associated encryption
    * key used when reading and writing files within the zone.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f5d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 1f51732..093d516 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -349,6 +349,7 @@ public class TestEncryptionZones {
   @Test
   public void testBasicOperations() throws Exception {
 
+    assertNotNull("key provider is not present", dfsAdmin.getKeyProvider());
     int numZones = 0;
     /* Number of EZs should be 0 if no EZ is created */
     assertEquals("Unexpected number of encryption zones!", numZones,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f5d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
index 717d79e..fe20c68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -27,8 +28,11 @@ import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
 import org.apache.hadoop.fs.BlockStoragePolicySpi;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.client.HdfsAdmin;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -172,4 +176,33 @@ public class TestHdfsAdmin {
     Assert.assertTrue(
         Sets.difference(policyNamesSet2, policyNamesSet1).isEmpty());
   }
+
+  private static String getKeyProviderURI() {
+    FileSystemTestHelper helper = new FileSystemTestHelper();
+    // Set up java key store
+    String testRoot = helper.getTestRootDir();
+    File testRootDir = new File(testRoot).getAbsoluteFile();
+    return JavaKeyStoreProvider.SCHEME_NAME + "://file" +
+        new Path(testRootDir.toString(), "test.jks").toUri();
+  }
+
+  @Test
+  public void testGetKeyProvider() throws IOException {
+    HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+    Assert.assertNull("should return null for an non-encrypted cluster",
+        hdfsAdmin.getKeyProvider());
+
+    shutDownCluster();
+
+    Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+        getKeyProviderURI());
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+    cluster.waitActive();
+    hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+
+    Assert.assertNotNull("should not return null for an encrypted cluster",
+        hdfsAdmin.getKeyProvider());
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message