hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject [1/2] hadoop git commit: Revert "HADOOP-13966 Add ability to start DDB local server in every test."
Date Fri, 24 Mar 2017 18:19:47 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 9521c96c6 -> ed15abaf3


Revert "HADOOP-13966 Add ability to start DDB local server in every test."

This reverts commit 9521c96c6a4f7227f51157336cba09156dea1cbc.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/737804b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/737804b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/737804b7

Branch: refs/heads/HADOOP-13345
Commit: 737804b742ddf8c86fd4f2789c4f0814583809a5
Parents: 9521c96
Author: Mingliang Liu <liuml07@apache.org>
Authored: Fri Mar 24 11:03:39 2017 -0700
Committer: Mingliang Liu <liuml07@apache.org>
Committed: Fri Mar 24 11:03:39 2017 -0700

----------------------------------------------------------------------
 hadoop-tools/hadoop-aws/pom.xml                 |  13 --
 .../fs/s3a/s3guard/DynamoDBClientFactory.java   |   4 +-
 .../apache/hadoop/fs/s3a/s3guard/S3Guard.java   |   4 +-
 .../site/markdown/tools/hadoop-aws/s3guard.md   |  18 +--
 .../apache/hadoop/fs/s3a/S3ATestConstants.java  |   1 -
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |   7 -
 .../s3a/s3guard/DynamoDBLocalClientFactory.java | 142 -------------------
 .../s3a/s3guard/TestDynamoDBMetadataStore.java  |  73 ++++++++--
 8 files changed, 72 insertions(+), 190 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 0606a3d..de9697e 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -312,19 +312,6 @@
       </properties>
     </profile>
 
-    <!-- Switch to DynamoDBLocal for s3guard. Has no effect unless s3guard is enabled
-->
-    <profile>
-      <id>dynamodblocal</id>
-      <activation>
-        <property>
-          <name>dynamodblocal</name>
-        </property>
-      </activation>
-      <properties>
-        <fs.s3a.s3guard.test.implementation>dynamodblocal</fs.s3a.s3guard.test.implementation>
-      </properties>
-    </profile>
-
     <!-- Switch s3guard from Authoritative=false to true
      Has no effect unless s3guard is enabled -->
     <profile>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
index c780189..c2fe6a0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
@@ -30,7 +30,6 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -44,8 +43,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet;
  *
  * Implementation should be configured for setting and getting configuration.
  */
-@InterfaceAudience.Private
-public interface DynamoDBClientFactory extends Configurable {
+interface DynamoDBClientFactory extends Configurable {
   Logger LOG = LoggerFactory.getLogger(DynamoDBClientFactory.class);
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
index cebc50e..9658ed9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.fs.s3a.s3guard;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -51,8 +50,7 @@ public final class S3Guard {
 
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
-  @VisibleForTesting
-  public static final String S3GUARD_DDB_CLIENT_FACTORY_IMPL =
+  static final String S3GUARD_DDB_CLIENT_FACTORY_IMPL =
       "fs.s3a.s3guard.ddb.client.factory.impl";
 
   static final Class<? extends DynamoDBClientFactory>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
index d8c0768..7619b2a 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
@@ -16,8 +16,6 @@
 
 **Experimental Feature**
 
-<!-- MACRO{toc|fromDepth=0|toDepth=5} -->
-
 ## Overview
 
 *S3Guard* is an experimental feature for the S3A client of the S3 Filesystem,
@@ -426,21 +424,17 @@ The basic strategy for testing S3Guard correctness consists of:
 
 All the S3A tests which work with a private repository can be configured to
 run with S3Guard by using the `s3guard` profile. When set, this will run
-all the tests with local memory for the metadata set to "non-authoritative" mode.
+all the tests with a local dynamo DB instance set to "non-authoritative" mode.
 
 ```bash
 mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard 
 ```
 
-When the `s3guard` profile is enabled, following profiles can be specified:
-
-* `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
-  not exist. You will have to pay the bills for DynamoDB web service.
-* `dynamodblocal`: use an in-memory DynamoDBLocal server instead of real AWS
-  DynamoDB web service; launch the server if it is not yet started; creating the
-  table if it does not exist. You won't be charged bills for using DynamoDB in
-  test. However, the DynamoDBLocal is a simulator of real AWS DynamoDB and is
-  maintained separately, so it may be stale.
+When the `s3guard` profile is enabled, are two other profiles which can be
+enabled
+ 
+* `dynamo` : use an AWS-hosted dynamo DB table, instead of a local table; creating
+the table if it does not exist.
 * `non-auth`: treat the s3guard metadata as authorative
 
 ```bash

http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index ccc28de..46bc9a6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -143,7 +143,6 @@ public interface S3ATestConstants {
   String TEST_S3GUARD_IMPLEMENTATION = TEST_S3GUARD_PREFIX + ".implementation";
   String TEST_S3GUARD_IMPLEMENTATION_LOCAL = "local";
   String TEST_S3GUARD_IMPLEMENTATION_DYNAMO = "dynamo";
-  String TEST_S3GUARD_IMPLEMENTATION_DYNAMODBLOCAL = "dynamodblocal";
   String TEST_S3GUARD_IMPLEMENTATION_NONE = "none";
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index f46f9ba..2e6a6e3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -25,10 +25,6 @@ import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory;
-import org.apache.hadoop.fs.s3a.s3guard.DynamoDBLocalClientFactory;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-
 import org.hamcrest.core.Is;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -370,9 +366,6 @@ public final class S3ATestUtils {
       case TEST_S3GUARD_IMPLEMENTATION_LOCAL:
         implClass = S3GUARD_METASTORE_LOCAL;
         break;
-      case TEST_S3GUARD_IMPLEMENTATION_DYNAMODBLOCAL:
-        conf.setClass(S3Guard.S3GUARD_DDB_CLIENT_FACTORY_IMPL,
-            DynamoDBLocalClientFactory.class, DynamoDBClientFactory.class);
       case TEST_S3GUARD_IMPLEMENTATION_DYNAMO:
         implClass = S3GUARD_METASTORE_DYNAMO;
         break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
deleted file mode 100644
index c584d68..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.File;
-import java.io.IOException;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
-import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
-import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
-import org.apache.hadoop.net.ServerSocketUtil;
-
-import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet;
-
-/**
- * A DynamoDBClientFactory implementation that creates AmazonDynamoDB clients
- * against an in-memory DynamoDBLocal server instance.
- *
- * You won't be charged bills for issuing any DynamoDB requests. However, the
- * DynamoDBLocal is considered a simulator of the DynamoDB web service, so it
- * may be stale or different. For example, the throttling is not yet supported
- * in DynamoDBLocal. This is for testing purpose only.
- *
- * To use this for creating DynamoDB client in tests:
- * 1. As all DynamoDBClientFactory implementations, this should be configured.
- * 2. The singleton DynamoDBLocal server instance is started automatically when
- *    creating the AmazonDynamoDB client for the first time. It still merits to
- *    launch the server before all the tests and fail fast if error happens.
- * 3. The sever can be stopped explicitly, which is not actually needed in tests
- *    as JVM termination will do that.
- *
- * @see DefaultDynamoDBClientFactory
- */
-public class DynamoDBLocalClientFactory extends Configured
-    implements DynamoDBClientFactory {
-
-  /** The DynamoDBLocal dynamoDBLocalServer instance for testing. */
-  private static DynamoDBProxyServer dynamoDBLocalServer;
-  private static String ddbEndpoint;
-
-  private static final String SYSPROP_SQLITE_LIB = "sqlite4java.library.path";
-
-  @Override
-  public AmazonDynamoDB createDynamoDBClient(String region) throws IOException {
-    startSingletonServer();
-
-    final Configuration conf = getConf();
-    final AWSCredentialsProvider credentials =
-        createAWSCredentialProviderSet(null, conf, null);
-    final ClientConfiguration awsConf =
-        DefaultS3ClientFactory.createAwsConf(conf);
-    // fail fast in case of service errors
-    awsConf.setMaxErrorRetry(3);
-    LOG.info("Creating DynamoDBLocal client using endpoint {}", ddbEndpoint);
-    return AmazonDynamoDBClientBuilder.standard()
-        .withCredentials(credentials)
-        .withClientConfiguration(awsConf)
-        .withEndpointConfiguration(
-            new AwsClientBuilder.EndpointConfiguration(ddbEndpoint, region))
-        .build();
-  }
-
-  /**
-   * Start a singleton in-memory DynamoDBLocal server if not started yet.
-   * @throws IOException if any error occurs
-   */
-  public synchronized static void startSingletonServer() throws IOException {
-    if (dynamoDBLocalServer != null) {
-      return;
-    }
-
-    // Set this property if it has not been set elsewhere
-    if (StringUtils.isEmpty(System.getProperty(SYSPROP_SQLITE_LIB))) {
-      String projectBuildDir = System.getProperty("project.build.directory");
-      if (StringUtils.isEmpty(projectBuildDir)) {
-        projectBuildDir = "target";
-      }
-      // sqlite4java lib should have been copied to $projectBuildDir/native-libs
-      System.setProperty(SYSPROP_SQLITE_LIB,
-          projectBuildDir + File.separator + "native-libs");
-      LOG.info("Setting {} -> {}",
-          SYSPROP_SQLITE_LIB, System.getProperty(SYSPROP_SQLITE_LIB));
-    }
-
-    try {
-      // Start an in-memory local DynamoDB instance
-      final String port = String.valueOf(ServerSocketUtil.getPort(0, 100));
-      dynamoDBLocalServer = ServerRunner.createServerFromCommandLineArgs(
-          new String[]{"-inMemory", "-port", port});
-      dynamoDBLocalServer.start();
-      ddbEndpoint = "http://localhost:" + port;
-      LOG.info("DynamoDBLocal singleton server was started at {}", ddbEndpoint);
-    } catch (Throwable t) {
-      String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint;
-      LOG.error(msg, t);
-      throw new IOException(msg, t);
-    }
-  }
-
-  /**
-   * Stop the in-memory DynamoDBLocal server if it is started.
-   * @throws IOException if any error occurs
-   */
-  public synchronized static void stopSingletonServer() throws IOException {
-    if (dynamoDBLocalServer != null) {
-      LOG.info("Shutting down the in-memory DynamoDBLocal server");
-      try {
-        dynamoDBLocalServer.stop();
-      } catch (Throwable t) {
-        String msg = "Error stopping DynamoDBLocal server at " + ddbEndpoint;
-        LOG.error(msg, t);
-        throw new IOException(msg, t);
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/737804b7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index ca5f95c..89fc507 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -24,17 +24,24 @@ import java.util.ArrayList;
 import java.util.Collection;
 
 import com.amazonaws.AmazonServiceException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
 import com.amazonaws.services.dynamodbv2.document.DynamoDB;
 import com.amazonaws.services.dynamodbv2.document.Item;
 import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
 import com.amazonaws.services.dynamodbv2.document.Table;
+import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
+import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
 import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
 import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
 import com.amazonaws.services.dynamodbv2.model.TableDescription;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.hadoop.fs.s3a.Tristate;
-
+import org.apache.log4j.Level;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -45,18 +52,23 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
 import org.apache.hadoop.fs.s3a.MockS3ClientFactory;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ClientFactory;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 
 import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet;
 import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
 import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
 import static org.apache.hadoop.test.LambdaTestUtils.*;
@@ -64,11 +76,11 @@ import static org.apache.hadoop.test.LambdaTestUtils.*;
 /**
  * Test that {@link DynamoDBMetadataStore} implements {@link MetadataStore}.
  *
- * In this unit test, we use an in-memory DynamoDBLocal server instead of real
- * AWS DynamoDB. An {@link S3AFileSystem} object is created and shared for
+ * In this unit test, we create an in-memory DynamoDBLocal server instance for
+ * all unit test cases.  You won't be charged bills for DynamoDB requests when
+ * you run this test.  An {@link S3AFileSystem} object is created and shared for
  * initializing {@link DynamoDBMetadataStore} objects.  There are no real S3
- * request issued as the underlying AWS S3Client is mocked.  You won't be
- * charged bills for AWS S3 or DynamoDB when you run this test.
+ * request issued as the underlying AWS S3Client is mocked.
  *
  * According to the base class, every test case will have independent contract
  * to create a new {@link DynamoDBMetadataStore} instance and initializes it.
@@ -85,6 +97,9 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
       VERSION_MARKER_PRIMARY_KEY = createVersionMarkerPrimaryKey(
       DynamoDBMetadataStore.VERSION_MARKER);
 
+  /** The DynamoDBLocal dynamoDBLocalServer instance for testing. */
+  private static DynamoDBProxyServer dynamoDBLocalServer;
+  private static String ddbEndpoint;
   /** The DynamoDB instance that can issue requests directly to server. */
   private static DynamoDB dynamoDB;
 
@@ -92,11 +107,22 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase
{
   public final Timeout timeout = new Timeout(60 * 1000);
 
   /**
-   * Start the in-memory DynamoDBLocal server and initializes s3 file system.
+   * Sets up the in-memory DynamoDBLocal server and initializes s3 file system.
    */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    DynamoDBLocalClientFactory.startSingletonServer();
+    GenericTestUtils.setLogLevel(DynamoDBMetadataStore.LOG, Level.ALL);
+    // sqlite4java library should have been copied to target/native-libs
+    System.setProperty("sqlite4java.library.path", "target/native-libs");
+
+    // Set up the in-memory local DynamoDB instance for all test cases
+    final String port = String.valueOf(ServerSocketUtil.getPort(0, 100));
+    dynamoDBLocalServer = ServerRunner.createServerFromCommandLineArgs(
+        new String[] {"-inMemory", "-port", port});
+    dynamoDBLocalServer.start();
+    ddbEndpoint = "http://localhost:" + port;
+    LOG.info("DynamoDBLocal for test was started at {}", ddbEndpoint);
+
     try {
       dynamoDB = new DynamoDBMSContract().getMetadataStore().getDynamoDB();
     } catch (AmazonServiceException e) {
@@ -114,7 +140,36 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase
{
     if (dynamoDB != null) {
       dynamoDB.shutdown();
     }
-    DynamoDBLocalClientFactory.stopSingletonServer();
+    if (dynamoDBLocalServer != null) {
+      LOG.info("Shutting down the in-memory local DynamoDB server");
+      try {
+        dynamoDBLocalServer.stop();
+      } catch (Exception e) {
+        final String msg = "Got exception to stop the DynamoDBLocal server. ";
+        LOG.error(msg, e);
+        fail(msg + e.getLocalizedMessage());
+      }
+    }
+  }
+
+  static class LocalDynamoDBClientFactory extends Configured
+      implements DynamoDBClientFactory {
+    @Override
+    public AmazonDynamoDB createDynamoDBClient(String region)
+        throws IOException {
+      final Configuration conf = getConf();
+      final AWSCredentialsProvider credentials =
+          createAWSCredentialProviderSet(null, conf, null);
+      final ClientConfiguration awsConf =
+          DefaultS3ClientFactory.createAwsConf(conf);
+      LOG.info("Creating AmazonDynamoDB client using endpoint {}", ddbEndpoint);
+      return AmazonDynamoDBClientBuilder.standard()
+          .withCredentials(credentials)
+          .withClientConfiguration(awsConf)
+          .withEndpointConfiguration(
+              new AwsClientBuilder.EndpointConfiguration(ddbEndpoint, region))
+          .build();
+    }
   }
 
   /**
@@ -135,7 +190,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
       conf.set(Constants.SECRET_KEY, "dummy-secret-key");
       conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true);
       conf.setClass(S3Guard.S3GUARD_DDB_CLIENT_FACTORY_IMPL,
-          DynamoDBLocalClientFactory.class, DynamoDBClientFactory.class);
+          LocalDynamoDBClientFactory.class, DynamoDBClientFactory.class);
 
       // always create new file system object for a test contract
       s3afs = (S3AFileSystem) FileSystem.newInstance(conf);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message