Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D9059200C5E for ; Sat, 8 Apr 2017 01:49:53 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D7876160BA2; Fri, 7 Apr 2017 23:49:53 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 031AC160B97 for ; Sat, 8 Apr 2017 01:49:52 +0200 (CEST) Received: (qmail 19388 invoked by uid 500); 7 Apr 2017 23:49:52 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 19376 invoked by uid 99); 7 Apr 2017 23:49:52 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Apr 2017 23:49:52 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DDEB4E080F; Fri, 7 Apr 2017 23:49:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mackrorysd@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HADOOP-14288. TestDynamoDBMEtadataStore is broken unless we can fail faster without a table version Date: Fri, 7 Apr 2017 23:49:51 +0000 (UTC) archived-at: Fri, 07 Apr 2017 23:49:54 -0000 Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 26e1eaa60 -> 13fafee1b HADOOP-14288. TestDynamoDBMEtadataStore is broken unless we can fail faster without a table version Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13fafee1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13fafee1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13fafee1 Branch: refs/heads/HADOOP-13345 Commit: 13fafee1be950cda8cef96d089e5ee4be223bba0 Parents: 26e1eaa Author: Sean Mackrory Authored: Fri Apr 7 17:48:42 2017 -0600 Committer: Sean Mackrory Committed: Fri Apr 7 17:48:50 2017 -0600 ---------------------------------------------------------------------- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 9 ++++---- .../fs/s3a/s3guard/MetadataStoreTestBase.java | 11 +++++++++- .../s3a/s3guard/TestDynamoDBMetadataStore.java | 23 ++++++++++++++++---- .../fs/s3a/s3guard/TestLocalMetadataStore.java | 11 +++++++++- .../fs/s3a/s3guard/TestNullMetadataStore.java | 5 +++++ 5 files changed, 49 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java index e0a171d..2b28c58 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java @@ -191,7 +191,7 @@ public class DynamoDBMetadataStore implements MetadataStore { private Configuration conf; private String username; - private RetryPolicy batchRetryPolicy; + private RetryPolicy dataAccessRetryPolicy; /** * A utility function to create DynamoDB instance. @@ -274,7 +274,7 @@ public class DynamoDBMetadataStore implements MetadataStore { private void setMaxRetries(Configuration config) { int maxRetries = config.getInt(S3GUARD_DDB_MAX_RETRIES, S3GUARD_DDB_MAX_RETRIES_DEFAULT); - batchRetryPolicy = RetryPolicies + dataAccessRetryPolicy = RetryPolicies .exponentialBackoffRetry(maxRetries, MIN_RETRY_SLEEP_MSEC, TimeUnit.MILLISECONDS); } @@ -517,7 +517,7 @@ public class DynamoDBMetadataStore implements MetadataStore { private void retryBackoff(int retryCount) throws IOException { try { // Our RetryPolicy ignores everything but retryCount here. - RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null, + RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null, retryCount, 0, true); if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) { throw new IOException( @@ -764,7 +764,7 @@ public class DynamoDBMetadataStore implements MetadataStore { Item versionMarker = table.getItem(versionMarkerKey); while (versionMarker == null) { try { - RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null, + RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null, retryCount, 0, true); if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) { break; @@ -775,6 +775,7 @@ public class DynamoDBMetadataStore implements MetadataStore { } catch (Exception e) { throw new IOException("initTable: Unexpected exception", e); } + retryCount++; versionMarker = table.getItem(versionMarkerKey); } return versionMarker; http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java index 36a5cc5..88df45e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.s3a.s3guard; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -63,12 +64,20 @@ public abstract class MetadataStoreTestBase extends Assert { private final long modTime = accessTime - 5000; /** - * Each test should override this. + * Each test should override this. Will use a new Configuration instance. * @return Contract which specifies the MetadataStore under test plus config. */ public abstract AbstractMSContract createContract() throws IOException; /** + * Each test should override this. + * @param conf Base configuration instance to use. + * @return Contract which specifies the MetadataStore under test plus config. + */ + public abstract AbstractMSContract createContract(Configuration conf) + throws IOException; + + /** * Tests assume that implementations will return recently set results. If * your implementation does not always hold onto metadata (e.g. LRU or * time-based expiry) you can override this to return false. http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java index a841e21..3584b54 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java @@ -126,7 +126,10 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { private final DynamoDBMetadataStore ms = new DynamoDBMetadataStore(); DynamoDBMSContract() throws IOException { - final Configuration conf = new Configuration(); + this(new Configuration()); + } + + DynamoDBMSContract(Configuration conf) throws IOException { // using mocked S3 clients conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, S3ClientFactory.class); @@ -160,6 +163,12 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { } @Override + public DynamoDBMSContract createContract(Configuration conf) throws + IOException { + return new DynamoDBMSContract(conf); + } + + @Override FileStatus basicFileStatus(Path path, int size, boolean isDir) throws IOException { String owner = UserGroupInformation.getCurrentUser().getShortUserName(); @@ -340,15 +349,21 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { */ @Test public void testTableVersionRequired() throws Exception { - final DynamoDBMetadataStore ddbms = createContract().getMetadataStore(); - String tableName = getFileSystem().getConf().get(Constants - .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); + Configuration conf = getFileSystem().getConf(); + int maxRetries = conf.getInt(Constants.S3GUARD_DDB_MAX_RETRIES, Constants + .S3GUARD_DDB_MAX_RETRIES_DEFAULT); + conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, 3); + + final DynamoDBMetadataStore ddbms = createContract(conf).getMetadataStore(); + String tableName = conf.get(Constants.S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); Table table = verifyTableInitialized(tableName); table.deleteItem(VERSION_MARKER_PRIMARY_KEY); // create existing table intercept(IOException.class, E_NO_VERSION_MARKER, () -> ddbms.initTable()); + + conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, maxRetries); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java index c3eb0d3..4cffc6f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java @@ -42,7 +42,10 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase { private FileSystem fs; private LocalMSContract() throws IOException { - Configuration config = new Configuration(); + this(new Configuration()); + } + + private LocalMSContract(Configuration config) throws IOException { config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR); fs = FileSystem.getLocal(config); } @@ -64,6 +67,12 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase { return new LocalMSContract(); } + @Override + public AbstractMSContract createContract(Configuration conf) throws + IOException { + return new LocalMSContract(conf); + } + @Test public void testClearByAncestor() { Map map = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java index 5b18686..5b19efa 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java @@ -55,4 +55,9 @@ public class TestNullMetadataStore extends MetadataStoreTestBase { public AbstractMSContract createContract() { return new NullMSContract(); } + + @Override + public AbstractMSContract createContract(Configuration conf) { + return createContract(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org