hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject hadoop git commit: HADOOP-13908. S3Guard: Existing tables may not be initialized correctly in DynamoDBMetadataStore. Contributed by Mingliang Liu.
Date Tue, 10 Jan 2017 00:08:26 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 e3f20027f -> a5cc315db


HADOOP-13908. S3Guard: Existing tables may not be initialized correctly in DynamoDBMetadataStore.
Contributed by Mingliang Liu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5cc315d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5cc315d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5cc315d

Branch: refs/heads/HADOOP-13345
Commit: a5cc315dbef15e8f708663d45800fdc957797cf2
Parents: e3f2002
Author: Chris Nauroth <cnauroth@apache.org>
Authored: Mon Jan 9 15:48:30 2017 -0800
Committer: Chris Nauroth <cnauroth@apache.org>
Committed: Mon Jan 9 15:48:30 2017 -0800

----------------------------------------------------------------------
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   | 83 ++++++++++++--------
 .../s3a/s3guard/TestDynamoDBMetadataStore.java  | 27 ++++++-
 2 files changed, 73 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5cc315d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 45ecaff..6ff0ee1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -190,10 +190,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
     // use the bucket as the DynamoDB table name if not specified in config
     tableName = conf.getTrimmed(S3GUARD_DDB_TABLE_NAME_KEY, bucket);
 
-    // create the table unless it's explicitly told not to do so
-    if (conf.getBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false)) {
-      createTable();
-    }
+    initTable();
   }
 
   /**
@@ -230,7 +227,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
     dynamoDB = new DynamoDB(dynamoDBClient);
     region = dynamoDBClient.getEndpointPrefix();
 
-    createTable();
+    initTable();
   }
 
   @Override
@@ -510,46 +507,64 @@ public class DynamoDBMetadataStore implements MetadataStore {
   /**
    * Create a table if it does not exist and wait for it to become active.
    *
-   * If a table with the intended name already exists, then it logs the
-   * {@link ResourceInUseException} and uses that table. The DynamoDB table
-   * creation API is asynchronous.  This method wait for the table to become
-   * active after sending the creation request, so overall, this method is
-   * synchronous, and the table is guaranteed to exist after this method
-   * returns successfully.
+   * If a table with the intended name already exists, then it uses that table.
+   * Otherwise, it will automatically create the table if the config
+   * {@link org.apache.hadoop.fs.s3a.Constants#S3GUARD_DDB_TABLE_CREATE_KEY} is
+   * enabled. The DynamoDB table creation API is asynchronous.  This method wait
+   * for the table to become active after sending the creation request, so
+   * overall, this method is synchronous, and the table is guaranteed to exist
+   * after this method returns successfully.
+   *
+   * @throws IOException if table does not exist and auto-creation is disabled;
+   * or any other I/O exception occurred.
    */
   @VisibleForTesting
-  void createTable() throws IOException {
+  void initTable() throws IOException {
     final ProvisionedThroughput capacity = new ProvisionedThroughput(
         conf.getLong(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY,
             S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT),
         conf.getLong(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY,
             S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT));
 
+    table = dynamoDB.getTable(tableName);
     try {
-      LOG.info("Creating DynamoDB table {} in region {}", tableName, region);
-      table = dynamoDB.createTable(new CreateTableRequest()
-          .withTableName(tableName)
-          .withKeySchema(keySchema())
-          .withAttributeDefinitions(attributeDefinitions())
-          .withProvisionedThroughput(capacity));
-    } catch (ResourceInUseException e) {
-      LOG.info("ResourceInUseException while creating DynamoDB table {} in "
-              + "region {}.  This may indicate that the table was created by "
-              + "another concurrent thread or process.",
-          tableName, region);
-      table = dynamoDB.getTable(tableName);
-    }
+      try {
+        table.describe();
+        LOG.debug("Using existing DynamoDB table {} in region {}",
+            tableName, region);
+      } catch (ResourceNotFoundException rnfe) {
+        if (conf.getBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false)) {
+          try {
+            LOG.info("Creating non-existent DynamoDB table {} in region {}",
+                tableName, region);
+            dynamoDB.createTable(new CreateTableRequest()
+                .withTableName(tableName)
+                .withKeySchema(keySchema())
+                .withAttributeDefinitions(attributeDefinitions())
+                .withProvisionedThroughput(capacity));
+          } catch (ResourceInUseException e) {
+            LOG.debug("ResourceInUseException while creating DynamoDB table {} "
+                    + "in region {}.  This may indicate that the table was "
+                    + "created by another concurrent thread or process.",
+                tableName, region);
+          }
+        } else {
+          throw new IOException("DynamoDB table '" + tableName + "' does not "
+              + "exist in region " + region + "; auto-creation is turned off");
+        }
+      }
 
-    try {
-      table.waitForActive();
-    } catch (InterruptedException e) {
-      LOG.warn("Interrupted while waiting for DynamoDB table {} active",
-          tableName, e);
-      Thread.currentThread().interrupt();
-      throw new InterruptedIOException("DynamoDB table '" + tableName + "'" +
-          " is not active yet in region " + region);
+      try {
+        table.waitForActive();
+      } catch (InterruptedException e) {
+        LOG.warn("Interrupted while waiting for DynamoDB table {} active",
+            tableName, e);
+        Thread.currentThread().interrupt();
+        throw new InterruptedIOException("DynamoDB table '" + tableName + "'" +
+            " is not active yet in region " + region);
+      }
     } catch (AmazonClientException e) {
-      throw translateException("createTable", (String) null, e);
+      throw translateException("initTable", (String) null, e);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5cc315d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index f88137b..fe38c12 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -214,7 +214,6 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
   public void testInitializeWithConfiguration() throws IOException {
     final String tableName = "testInitializeWithConfiguration";
     final Configuration conf = new Configuration();
-    String a = conf.get(Constants.S3GUARD_DDB_ENDPOINT_KEY);
     try {
       DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
       ddbms.initialize(conf);
@@ -234,6 +233,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
     // config credentials
     conf.set(Constants.ACCESS_KEY, "dummy-access-key");
     conf.set(Constants.SECRET_KEY, "dummy-secret-key");
+    conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true);
     try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
       ddbms.initialize(conf);
       verifyTableInitialized(tableName);
@@ -312,15 +312,36 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase
{
   }
 
   @Test
-  public void testCreateExistingTable() throws IOException {
+  public void testInitExistingTable() throws IOException {
     final DynamoDBMetadataStore ddbms = createContract().getMetadataStore();
     verifyTableInitialized(BUCKET);
     // create existing table
-    ddbms.createTable();
+    ddbms.initTable();
     verifyTableInitialized(BUCKET);
   }
 
   /**
+   * Test that initTable fails with IOException when table does not exist and
+   * table auto-creation is disabled.
+   */
+  @Test
+  public void testFailNonexistentTable() throws IOException {
+    final String tableName = "testFailNonexistentTable";
+    final DynamoDBMSContract contract = createContract();
+    final S3AFileSystem s3afs = contract.getFileSystem();
+    final Configuration conf = s3afs.getConf();
+    conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName);
+    conf.unset(Constants.S3GUARD_DDB_TABLE_CREATE_KEY);
+    try {
+      final DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
+      ddbms.initialize(s3afs);
+      fail("Should have failed as table does not exist and table auto-creation "
+          + "is disabled");
+    } catch (IOException ignored) {
+    }
+  }
+
+  /**
    * Test cases about root directory as it is not in the DynamoDB table.
    */
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message