hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject hadoop git commit: HADOOP-14802. Add support for using container saskeys for all accesses. Contributed by Sivaguru Sankaridurg
Date Wed, 30 Aug 2017 21:41:15 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 7dd5bb949 -> 25e9c2960


HADOOP-14802. Add support for using container saskeys for all accesses.
Contributed by Sivaguru Sankaridurg

(cherry picked from commit 021974f4cb1eb8cf530dabc95039fed46e66a43d)

Conflicts:
	hadoop-common-project/hadoop-common/src/main/resources/core-default.xml


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25e9c296
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25e9c296
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25e9c296

Branch: refs/heads/branch-2
Commit: 25e9c2960870ee6397f02f7bd810905ec7b3954a
Parents: 7dd5bb9
Author: Steve Loughran <stevel@apache.org>
Authored: Wed Aug 30 22:40:50 2017 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Wed Aug 30 22:40:50 2017 +0100

----------------------------------------------------------------------
 .../src/main/resources/core-default.xml         | 10 +++++
 .../conf/TestCommonConfigurationFields.java     |  1 +
 .../fs/azure/SecureStorageInterfaceImpl.java    | 29 +++++++++----
 .../hadoop-azure/src/site/markdown/index.md     |  9 ++++
 ...stNativeAzureFSAuthWithBlobSpecificKeys.java | 44 ++++++++++++++++++++
 5 files changed, 84 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/25e9c296/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index c2c6db5..708d7f8 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1385,6 +1385,16 @@
   </description>
 </property>
 
+<property>
+  <name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
+  <value>true</value>
+  <description>
+    Use container saskey for access to all blobs within the container.
+    Blob-specific saskeys are not used when this setting is enabled.
+    This setting provides better performance compared to blob-specific saskeys.
+  </description>
+</property>
+
   <!-- Azure Data Lake File System Configurations -->
 
   <property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25e9c296/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 35c056c..2a6cdc2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -125,6 +125,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase
{
     xmlPropsToSkipCompare.add("fs.azure.secure.mode");
     xmlPropsToSkipCompare.add("fs.azure.authorization");
     xmlPropsToSkipCompare.add("fs.azure.authorization.caching.enable");
+    xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
     xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
 
     // ADL properties are in a different subtree

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25e9c296/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
index 3d33453..5dbb6bc 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
@@ -71,6 +71,13 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
   private String storageAccount;
   private RetryPolicyFactory retryPolicy;
   private int timeoutIntervalInMs;
+  private boolean useContainerSasKeyForAllAccess;
+
+  /**
+   * Configuration key to specify if containerSasKey should be used for all accesses
+   */
+  public static final String KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS =
+      "fs.azure.saskey.usecontainersaskeyforallaccess";
 
   public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
       Configuration conf) throws SecureModeException {
@@ -88,6 +95,7 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
       }
       this.sasKeyGenerator = remoteSasKeyGenerator;
     }
+    this.useContainerSasKeyForAllAccess = conf.getBoolean(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS,
true);
   }
 
   @Override
@@ -145,7 +153,9 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
       if (timeoutIntervalInMs > 0) {
         container.getServiceClient().getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs);
       }
-      return new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
+      return (useContainerSasKeyForAllAccess)
+          ? new SASCloudBlobContainerWrapperImpl(storageAccount, container, null)
+          : new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
     } catch (SASKeyGenerationException sasEx) {
       String errorMsg = "Encountered SASKeyGeneration exception while "
           + "generating SAS Key for container : " + name
@@ -226,12 +236,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
     public CloudBlobWrapper getBlockBlobReference(String relativePath)
         throws URISyntaxException, StorageException {
       try {
-        CloudBlockBlob blob = new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(
-                storageAccount, getName(), relativePath));
+        CloudBlockBlob blob = (sasKeyGenerator!=null)
+          ? new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(),
relativePath))
+          : container.getBlockBlobReference(relativePath);
         blob.getServiceClient().setDefaultRequestOptions(
                 container.getServiceClient().getDefaultRequestOptions());
-        return new SASCloudBlockBlobWrapperImpl(
-                blob);
+        return new SASCloudBlockBlobWrapperImpl(blob);
       } catch (SASKeyGenerationException sasEx) {
         String errorMsg = "Encountered SASKeyGeneration exception while "
             + "generating SAS Key for relativePath : " + relativePath
@@ -245,12 +255,13 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
     public CloudBlobWrapper getPageBlobReference(String relativePath)
         throws URISyntaxException, StorageException {
       try {
-        CloudPageBlob blob = new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(
-                storageAccount, getName(), relativePath));
+        CloudPageBlob blob   = (sasKeyGenerator!=null)
+          ? new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(),
relativePath))
+          : container.getPageBlobReference(relativePath);
+
         blob.getServiceClient().setDefaultRequestOptions(
                 container.getServiceClient().getDefaultRequestOptions());
-        return new SASCloudPageBlobWrapperImpl(
-                blob);
+        return new SASCloudPageBlobWrapperImpl(blob);
       } catch (SASKeyGenerationException sasEx) {
         String errorMsg = "Encountered SASKeyGeneration exception while "
             + "generating SAS Key for relativePath : " + relativePath

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25e9c296/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 9b3e360..3b42fc2 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -473,6 +473,15 @@ The maximum number of entries that that cache can hold can be customized
using t
     </property>
 ```
 
+ Use container saskey for access to all blobs within the container.
+ Blob-specific saskeys are not used when this setting is enabled.
+ This setting provides better performance compared to blob-specific saskeys.
+ ```
+    <property>
+      <name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
+      <value>true</value>
+    </property>
+```
 ## Testing the hadoop-azure Module
 
 The hadoop-azure module includes a full suite of unit tests.  Most of the tests

http://git-wip-us.apache.org/repos/asf/hadoop/blob/25e9c296/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
new file mode 100644
index 0000000..6149154
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS;
+
+/**
+ * Test class to hold all WASB authorization tests that use blob-specific keys
+ * to access storage.
+ */
+public class TestNativeAzureFSAuthWithBlobSpecificKeys
+    extends TestNativeAzureFileSystemAuthorizationWithOwner {
+
+  @Override
+  public Configuration getConfiguration() {
+    Configuration conf = super.getConfiguration();
+    conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false");
+    return conf;
+  }
+
+  @Override
+  protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+    Configuration conf = getConfiguration();
+    return AzureBlobStorageTestAccount.create(conf);
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message