Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D9AC0200CB3 for ; Mon, 26 Jun 2017 14:49:43 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D88BE160BF5; Mon, 26 Jun 2017 12:49:43 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 01E89160BD9 for ; Mon, 26 Jun 2017 14:49:42 +0200 (CEST) Received: (qmail 88709 invoked by uid 500); 26 Jun 2017 12:49:38 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 88455 invoked by uid 99); 26 Jun 2017 12:49:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 26 Jun 2017 12:49:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D72E3E96E8; Mon, 26 Jun 2017 12:49:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: stevel@apache.org To: common-commits@hadoop.apache.org Date: Mon, 26 Jun 2017 12:50:19 -0000 Message-Id: In-Reply-To: <58008a2722104227ad1d7d4cdc6db2b1@git.apache.org> References: <58008a2722104227ad1d7d4cdc6db2b1@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [45/50] hadoop git commit: HADOOP-14547. [WASB] the configured retry policy is not used for all storage operations. Contributed by Thomas. archived-at: Mon, 26 Jun 2017 12:49:44 -0000 HADOOP-14547. [WASB] the configured retry policy is not used for all storage operations. Contributed by Thomas. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6bd73c6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6bd73c6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6bd73c6 Branch: refs/heads/HADOOP-13345 Commit: c6bd73c6c5760c3a52203e6a51628587ceec6896 Parents: 0b77262 Author: Steve Loughran Authored: Fri Jun 23 21:38:41 2017 +0100 Committer: Steve Loughran Committed: Fri Jun 23 21:38:41 2017 +0100 ---------------------------------------------------------------------- .../fs/azure/AzureNativeFileSystemStore.java | 11 ++----- .../fs/azure/SecureStorageInterfaceImpl.java | 30 +++++++++++++++----- .../hadoop/fs/azure/StorageInterfaceImpl.java | 28 ++++++++++++++---- 3 files changed, 49 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6bd73c6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 534919e..f76b44a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -481,6 +481,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { } } + // Configure Azure storage session. + configureAzureStorageSession(); + // Start an Azure storage session. // createAzureStorageSession(); @@ -792,9 +795,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { // Accessing the storage server unauthenticated using // anonymous credentials. isAnonymousCredentials = true; - - // Configure Azure storage session. - configureAzureStorageSession(); } private void connectUsingCredentials(String accountName, @@ -820,9 +820,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { // Can only create container if using account key credentials canCreateOrModifyContainer = credentials instanceof StorageCredentialsAccountAndKey; - - // Configure Azure storage session. - configureAzureStorageSession(); } /** @@ -848,8 +845,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { rootDirectory = container.getDirectoryReference(""); canCreateOrModifyContainer = true; - - configureAzureStorageSession(); tolerateOobAppends = false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6bd73c6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java index 650149a..810aacf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java @@ -69,6 +69,8 @@ public class SecureStorageInterfaceImpl extends StorageInterface { public static final String SAS_ERROR_CODE = "SAS Error"; private SASKeyGeneratorInterface sasKeyGenerator; private String storageAccount; + private RetryPolicyFactory retryPolicy; + private int timeoutIntervalInMs; public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode, Configuration conf) throws SecureModeException { @@ -90,10 +92,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface { @Override public void setTimeoutInMs(int timeoutInMs) { + timeoutIntervalInMs = timeoutInMs; } @Override public void setRetryPolicyFactory(RetryPolicyFactory retryPolicyFactory) { + retryPolicy = retryPolicyFactory; } @Override @@ -133,9 +137,15 @@ public class SecureStorageInterfaceImpl extends StorageInterface { throws URISyntaxException, StorageException { try { - return new SASCloudBlobContainerWrapperImpl(storageAccount, - new CloudBlobContainer(sasKeyGenerator.getContainerSASUri( - storageAccount, name)), sasKeyGenerator); + CloudBlobContainer container = new CloudBlobContainer(sasKeyGenerator.getContainerSASUri( + storageAccount, name)); + if (retryPolicy != null) { + container.getServiceClient().getDefaultRequestOptions().setRetryPolicyFactory(retryPolicy); + } + if (timeoutIntervalInMs > 0) { + container.getServiceClient().getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs); + } + return new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator); } catch (SASKeyGenerationException sasEx) { String errorMsg = "Encountered SASKeyGeneration exception while " + "generating SAS Key for container : " + name @@ -216,9 +226,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface { public CloudBlobWrapper getBlockBlobReference(String relativePath) throws URISyntaxException, StorageException { try { + CloudBlockBlob blob = new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri( + storageAccount, getName(), relativePath)); + blob.getServiceClient().setDefaultRequestOptions( + container.getServiceClient().getDefaultRequestOptions()); return new SASCloudBlockBlobWrapperImpl( - new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri( - storageAccount, getName(), relativePath))); + blob); } catch (SASKeyGenerationException sasEx) { String errorMsg = "Encountered SASKeyGeneration exception while " + "generating SAS Key for relativePath : " + relativePath @@ -232,9 +245,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface { public CloudBlobWrapper getPageBlobReference(String relativePath) throws URISyntaxException, StorageException { try { + CloudPageBlob blob = new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri( + storageAccount, getName(), relativePath)); + blob.getServiceClient().setDefaultRequestOptions( + container.getServiceClient().getDefaultRequestOptions()); return new SASCloudPageBlobWrapperImpl( - new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri( - storageAccount, getName(), relativePath))); + blob); } catch (SASKeyGenerationException sasEx) { String errorMsg = "Encountered SASKeyGeneration exception while " + "generating SAS Key for relativePath : " + relativePath http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6bd73c6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java index 367cd04..491a0d0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java @@ -60,32 +60,50 @@ import com.microsoft.azure.storage.blob.PageRange; @InterfaceAudience.Private class StorageInterfaceImpl extends StorageInterface { private CloudBlobClient serviceClient; + private RetryPolicyFactory retryPolicyFactory; + private int timeoutIntervalInMs; + + private void updateRetryPolicy() { + if (serviceClient != null && retryPolicyFactory != null) { + serviceClient.getDefaultRequestOptions().setRetryPolicyFactory(retryPolicyFactory); + } + } + + private void updateTimeoutInMs() { + if (serviceClient != null && timeoutIntervalInMs > 0) { + serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs); + } + } @Override public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) { - serviceClient.getDefaultRequestOptions().setRetryPolicyFactory( - retryPolicyFactory); + this.retryPolicyFactory = retryPolicyFactory; + updateRetryPolicy(); } @Override public void setTimeoutInMs(int timeoutInMs) { - serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs( - timeoutInMs); + timeoutIntervalInMs = timeoutInMs; + updateTimeoutInMs(); } @Override public void createBlobClient(CloudStorageAccount account) { serviceClient = account.createCloudBlobClient(); + updateRetryPolicy(); + updateTimeoutInMs(); } @Override public void createBlobClient(URI baseUri) { - serviceClient = new CloudBlobClient(baseUri); + createBlobClient(baseUri, (StorageCredentials)null); } @Override public void createBlobClient(URI baseUri, StorageCredentials credentials) { serviceClient = new CloudBlobClient(baseUri, credentials); + updateRetryPolicy(); + updateTimeoutInMs(); } @Override --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org