hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mackror...@apache.org
Subject hadoop git commit: HADOOP-14288. TestDynamoDBMEtadataStore is broken unless we can fail faster without a table version
Date Fri, 07 Apr 2017 23:49:51 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 26e1eaa60 -> 13fafee1b


HADOOP-14288. TestDynamoDBMEtadataStore is broken unless we can fail faster without a table
version


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13fafee1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13fafee1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13fafee1

Branch: refs/heads/HADOOP-13345
Commit: 13fafee1be950cda8cef96d089e5ee4be223bba0
Parents: 26e1eaa
Author: Sean Mackrory <mackrorysd@apache.org>
Authored: Fri Apr 7 17:48:42 2017 -0600
Committer: Sean Mackrory <mackrorysd@apache.org>
Committed: Fri Apr 7 17:48:50 2017 -0600

----------------------------------------------------------------------
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   |  9 ++++----
 .../fs/s3a/s3guard/MetadataStoreTestBase.java   | 11 +++++++++-
 .../s3a/s3guard/TestDynamoDBMetadataStore.java  | 23 ++++++++++++++++----
 .../fs/s3a/s3guard/TestLocalMetadataStore.java  | 11 +++++++++-
 .../fs/s3a/s3guard/TestNullMetadataStore.java   |  5 +++++
 5 files changed, 49 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index e0a171d..2b28c58 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -191,7 +191,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
   private Configuration conf;
   private String username;
 
-  private RetryPolicy batchRetryPolicy;
+  private RetryPolicy dataAccessRetryPolicy;
 
   /**
    * A utility function to create DynamoDB instance.
@@ -274,7 +274,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
   private void setMaxRetries(Configuration config) {
     int maxRetries = config.getInt(S3GUARD_DDB_MAX_RETRIES,
         S3GUARD_DDB_MAX_RETRIES_DEFAULT);
-    batchRetryPolicy = RetryPolicies
+    dataAccessRetryPolicy = RetryPolicies
         .exponentialBackoffRetry(maxRetries, MIN_RETRY_SLEEP_MSEC,
             TimeUnit.MILLISECONDS);
   }
@@ -517,7 +517,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
   private void retryBackoff(int retryCount) throws IOException {
     try {
       // Our RetryPolicy ignores everything but retryCount here.
-      RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null,
+      RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null,
           retryCount, 0, true);
       if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
         throw new IOException(
@@ -764,7 +764,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
     Item versionMarker = table.getItem(versionMarkerKey);
     while (versionMarker == null) {
       try {
-        RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null,
+        RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null,
             retryCount, 0, true);
         if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
           break;
@@ -775,6 +775,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
       } catch (Exception e) {
         throw new IOException("initTable: Unexpected exception", e);
       }
+      retryCount++;
       versionMarker = table.getItem(versionMarkerKey);
     }
     return versionMarker;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 36a5cc5..88df45e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.s3a.s3guard;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -63,12 +64,20 @@ public abstract class MetadataStoreTestBase extends Assert {
   private final long modTime = accessTime - 5000;
 
   /**
-   * Each test should override this.
+   * Each test should override this.  Will use a new Configuration instance.
    * @return Contract which specifies the MetadataStore under test plus config.
    */
   public abstract AbstractMSContract createContract() throws IOException;
 
   /**
+   * Each test should override this.
+   * @param conf Base configuration instance to use.
+   * @return Contract which specifies the MetadataStore under test plus config.
+   */
+  public abstract AbstractMSContract createContract(Configuration conf)
+      throws IOException;
+
+  /**
    * Tests assume that implementations will return recently set results.  If
    * your implementation does not always hold onto metadata (e.g. LRU or
    * time-based expiry) you can override this to return false.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index a841e21..3584b54 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -126,7 +126,10 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase
{
     private final DynamoDBMetadataStore ms = new DynamoDBMetadataStore();
 
     DynamoDBMSContract() throws IOException {
-      final Configuration conf = new Configuration();
+        this(new Configuration());
+    }
+
+    DynamoDBMSContract(Configuration conf) throws IOException {
       // using mocked S3 clients
       conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
           S3ClientFactory.class);
@@ -160,6 +163,12 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase
{
   }
 
   @Override
+  public DynamoDBMSContract createContract(Configuration conf) throws
+      IOException {
+    return new DynamoDBMSContract(conf);
+  }
+
+  @Override
   FileStatus basicFileStatus(Path path, int size, boolean isDir)
       throws IOException {
     String owner = UserGroupInformation.getCurrentUser().getShortUserName();
@@ -340,15 +349,21 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase
{
    */
   @Test
   public void testTableVersionRequired() throws Exception {
-    final DynamoDBMetadataStore ddbms = createContract().getMetadataStore();
-    String tableName = getFileSystem().getConf().get(Constants
-        .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
+    Configuration conf = getFileSystem().getConf();
+    int maxRetries = conf.getInt(Constants.S3GUARD_DDB_MAX_RETRIES, Constants
+        .S3GUARD_DDB_MAX_RETRIES_DEFAULT);
+    conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, 3);
+
+    final DynamoDBMetadataStore ddbms = createContract(conf).getMetadataStore();
+    String tableName = conf.get(Constants.S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
     Table table = verifyTableInitialized(tableName);
     table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
 
     // create existing table
     intercept(IOException.class, E_NO_VERSION_MARKER,
         () -> ddbms.initTable());
+
+    conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, maxRetries);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
index c3eb0d3..4cffc6f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -42,7 +42,10 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
     private FileSystem fs;
 
     private LocalMSContract() throws IOException {
-      Configuration config = new Configuration();
+      this(new Configuration());
+    }
+
+    private LocalMSContract(Configuration config) throws IOException {
       config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR);
       fs = FileSystem.getLocal(config);
     }
@@ -64,6 +67,12 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
     return new LocalMSContract();
   }
 
+  @Override
+  public AbstractMSContract createContract(Configuration conf) throws
+      IOException {
+    return new LocalMSContract(conf);
+  }
+
   @Test
   public void testClearByAncestor() {
     Map<Path, String> map = new HashMap<>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
index 5b18686..5b19efa 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
@@ -55,4 +55,9 @@ public class TestNullMetadataStore extends MetadataStoreTestBase {
   public AbstractMSContract createContract() {
     return new NullMSContract();
   }
+
+  @Override
+  public AbstractMSContract createContract(Configuration conf) {
+    return createContract();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message