Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 03DF318C24 for ; Tue, 3 Nov 2015 21:09:29 +0000 (UTC) Received: (qmail 16221 invoked by uid 500); 3 Nov 2015 21:09:19 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 15786 invoked by uid 500); 3 Nov 2015 21:09:19 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 14533 invoked by uid 99); 3 Nov 2015 21:09:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Nov 2015 21:09:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7421BE055E; Tue, 3 Nov 2015 21:09:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jing9@apache.org To: common-commits@hadoop.apache.org Date: Tue, 03 Nov 2015 21:09:55 -0000 Message-Id: <9eaaf93410c04bc69bda0d96778080b0@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [39/50] hadoop git commit: HADOOP-12508. delete fails with exception when lease is held on blob. Contributed by Gaurav Kanade. HADOOP-12508. delete fails with exception when lease is held on blob. Contributed by Gaurav Kanade. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e7dcab1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e7dcab1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e7dcab1 Branch: refs/heads/HDFS-8966 Commit: 9e7dcab185abf2fdabb28f2799b9952b5664a4b0 Parents: 3ce0a65 Author: cnauroth Authored: Mon Nov 2 10:21:39 2015 -0800 Committer: cnauroth Committed: Mon Nov 2 10:21:39 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../fs/azure/AzureNativeFileSystemStore.java | 32 +++++++- .../hadoop/fs/azure/SelfRenewingLease.java | 5 +- .../fs/azure/TestNativeAzureFileSystemLive.java | 86 ++++++++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c8d60b0..1a9c93c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1307,6 +1307,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12533. Introduce FileNotFoundException in WASB for read and seek API. (Dushyanth via cnauroth) + HADOOP-12508. delete fails with exception when lease is held on blob. + (Gaurav Kanade via cnauroth) + OPTIMIZATIONS HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString() http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 6412714..69ece4a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -2370,7 +2370,37 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { @Override public void delete(String key) throws IOException { - delete(key, null); + try { + delete(key, null); + } catch (IOException e) { + Throwable t = e.getCause(); + if(t != null && t instanceof StorageException) { + StorageException se = (StorageException) t; + if(se.getErrorCode().equals(("LeaseIdMissing"))){ + SelfRenewingLease lease = null; + try { + lease = acquireLease(key); + delete(key, lease); + } catch (AzureException e3) { + LOG.warn("Got unexpected exception trying to acquire lease on " + + key + "." + e3.getMessage()); + throw e3; + } finally { + try { + if(lease != null){ + lease.free(); + } + } catch (Exception e4){ + LOG.error("Unable to free lease on " + key, e4); + } + } + } else { + throw e; + } + } else { + throw e; + } + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java index 06f32ce..900d730 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java @@ -22,6 +22,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper; +import com.google.common.annotations.VisibleForTesting; + import com.microsoft.azure.storage.AccessCondition; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlob; @@ -61,7 +63,8 @@ public class SelfRenewingLease { // Time to wait to retry getting the lease in milliseconds - private static final int LEASE_ACQUIRE_RETRY_INTERVAL = 2000; + @VisibleForTesting + static final int LEASE_ACQUIRE_RETRY_INTERVAL = 2000; public SelfRenewingLease(CloudBlobWrapper blobWrapper) throws StorageException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java index b033460..721cb5f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java @@ -21,10 +21,16 @@ package org.apache.hadoop.fs.azure; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; + +import java.util.concurrent.CountDownLatch; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; + import org.junit.Test; +import com.microsoft.azure.storage.StorageException; + /* * Tests the Native Azure file system (WASB) against an actual blob store if * provided in the environment. @@ -38,6 +44,86 @@ public class TestNativeAzureFileSystemLive extends } /** + * Tests fs.delete() function to delete a blob when another blob is holding a + * lease on it. Delete if called without a lease should fail if another process + * is holding a lease and throw appropriate exception + * This is a scenario that would happen in HMaster startup when it tries to + * clean up the temp dirs while the HMaster process which was killed earlier + * held lease on the blob when doing some DDL operation + */ + @Test + public void testDeleteThrowsExceptionWithLeaseExistsErrorMessage() + throws Exception { + LOG.info("Starting test"); + final String FILE_KEY = "fileWithLease"; + // Create the file + Path path = new Path(FILE_KEY); + fs.create(path); + assertTrue(fs.exists(path)); + NativeAzureFileSystem nfs = (NativeAzureFileSystem)fs; + final String fullKey = nfs.pathToKey(nfs.makeAbsolute(path)); + final AzureNativeFileSystemStore store = nfs.getStore(); + + // Acquire the lease on the file in a background thread + final CountDownLatch leaseAttemptComplete = new CountDownLatch(1); + final CountDownLatch beginningDeleteAttempt = new CountDownLatch(1); + Thread t = new Thread() { + @Override + public void run() { + // Acquire the lease and then signal the main test thread. + SelfRenewingLease lease = null; + try { + lease = store.acquireLease(fullKey); + LOG.info("Lease acquired: " + lease.getLeaseID()); + } catch (AzureException e) { + LOG.warn("Lease acqusition thread unable to acquire lease", e); + } finally { + leaseAttemptComplete.countDown(); + } + + // Wait for the main test thread to signal it will attempt the delete. + try { + beginningDeleteAttempt.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Keep holding the lease past the lease acquisition retry interval, so + // the test covers the case of delete retrying to acquire the lease. + try { + Thread.sleep(SelfRenewingLease.LEASE_ACQUIRE_RETRY_INTERVAL * 3); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + + try { + if (lease != null){ + LOG.info("Freeing lease"); + lease.free(); + } + } catch (StorageException se) { + LOG.warn("Unable to free lease.", se); + } + } + }; + + // Start the background thread and wait for it to signal the lease is held. + t.start(); + try { + leaseAttemptComplete.await(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + + // Try to delete the same file + beginningDeleteAttempt.countDown(); + store.delete(fullKey); + + // At this point file SHOULD BE DELETED + assertFalse(fs.exists(path)); + } + + /** * Check that isPageBlobKey works as expected. This assumes that * in the test configuration, the list of supported page blob directories * only includes "pageBlobs". That's why this test is made specific