hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject hadoop git commit: HADOOP-14125 s3guard tool tests aren't isolated; can't run in parallel. Contributed by Sean Mackrory.
Date Wed, 01 Mar 2017 10:55:55 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 f41d9b024 -> 0942c9f96


HADOOP-14125 s3guard tool tests aren't isolated; can't run in parallel.
Contributed by Sean Mackrory.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0942c9f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0942c9f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0942c9f9

Branch: refs/heads/HADOOP-13345
Commit: 0942c9f96d401936774ed659a629945bedbc8276
Parents: f41d9b0
Author: Steve Loughran <stevel@apache.org>
Authored: Wed Mar 1 10:55:19 2017 +0000
Committer: Steve Loughran <stevel@apache.org>
Committed: Wed Mar 1 10:55:19 2017 +0000

----------------------------------------------------------------------
 .../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java    |  2 +-
 .../hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java  | 14 +++++---------
 .../hadoop/fs/s3a/s3guard/TestS3GuardTool.java      | 16 +++++++++-------
 3 files changed, 15 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0942c9f9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index fb0efd1..c69bade 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -64,7 +64,7 @@ public class ITestS3GuardToolDynamoDB extends S3GuardToolTestBase {
   public void testDynamoDBInitDestroyCycle() throws IOException,
       InterruptedException {
     String testTableName = "testDynamoDBInitDestroy" + new Random().nextInt();
-    String testS3Url = getTestPath(testTableName);
+    String testS3Url = path(testTableName).toString();
     S3AFileSystem fs = getFs();
     DynamoDB db = null;
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0942c9f9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
index 666ec8e..0217a87 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.s3guard;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
 import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
@@ -39,7 +40,7 @@ import static org.junit.Assert.assertEquals;
 /**
  * Common functionality for S3GuardTool test cases.
  */
-public abstract class S3GuardToolTestBase {
+public abstract class S3GuardToolTestBase extends AbstractS3ATestBase {
 
   protected static final String OWNER = "hdfs";
 
@@ -59,11 +60,6 @@ public abstract class S3GuardToolTestBase {
     return fs;
   }
 
-  /** Get test path of s3. */
-  protected String getTestPath(String path) {
-    return fs.qualify(new Path(path)).toString();
-  }
-
   protected abstract MetadataStore newMetadataStore();
 
   @Before
@@ -125,7 +121,7 @@ public abstract class S3GuardToolTestBase {
 
   private void testPruneCommand(Configuration cmdConf, String[] args)
       throws Exception {
-    Path parent = new Path(getTestPath("/prune-cli"));
+    Path parent = path("prune-cli");
     try {
       fs.mkdirs(parent);
 
@@ -148,7 +144,7 @@ public abstract class S3GuardToolTestBase {
 
   @Test
   public void testPruneCommandCLI() throws Exception {
-    String testPath = getTestPath("testPruneCommandCLI");
+    String testPath = path("testPruneCommandCLI").toString();
     testPruneCommand(fs.getConf(), new String[]{"prune", "-S", "1", testPath});
   }
 
@@ -156,7 +152,7 @@ public abstract class S3GuardToolTestBase {
   public void testPruneCommandConf() throws Exception {
     conf.setLong(Constants.S3GUARD_CLI_PRUNE_AGE,
         TimeUnit.SECONDS.toMillis(1));
-    String testPath = getTestPath("testPruneCommandConf");
+    String testPath = path("testPruneCommandConf").toString();
     testPruneCommand(conf, new String[]{"prune", testPath});
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0942c9f9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
index 6a39d7b..09c6f6b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
@@ -51,8 +51,9 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
   public void testImportCommand() throws IOException {
     S3AFileSystem fs = getFs();
     MetadataStore ms = getMetadataStore();
-    fs.mkdirs(new Path("/test"));
-    Path dir = new Path("/test/a");
+    Path parent = path("test-import");
+    fs.mkdirs(parent);
+    Path dir = new Path(parent, "a");
     fs.mkdirs(dir);
     for (int i = 0; i < 10; i++) {
       String child = String.format("file-%d", i);
@@ -65,10 +66,10 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
     cmd.setMetadataStore(ms);
 
     assertEquals("Import command did not exit successfully - see output",
-        SUCCESS, cmd.run(new String[]{"import", getTestPath("/test/a")}));
+        SUCCESS, cmd.run(new String[]{"import", dir.toString()}));
 
     DirListingMetadata children =
-        ms.listChildren(new Path(getTestPath("/test/a")));
+        ms.listChildren(dir);
     assertEquals("Unexpected number of paths imported", 10, children
         .getListing().size());
     // assertTrue(children.isAuthoritative());
@@ -81,8 +82,8 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
     Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
     Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
 
-    String testPath = getTestPath("/test-diff");
-    mkdirs(new Path(testPath), true, true);
+    Path testPath = path("test-diff");
+    mkdirs(testPath, true, true);
 
     Path msOnlyPath = new Path(testPath, "ms_only");
     mkdirs(msOnlyPath, false, true);
@@ -107,7 +108,8 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
     Diff cmd = new Diff(fs.getConf());
     cmd.setMetadataStore(ms);
     assertEquals("Diff command did not exit successfully - see output", SUCCESS,
-        cmd.run(new String[]{"diff", "-m", "local://metadata", testPath}, out));
+        cmd.run(new String[]{"diff", "-m", "local://metadata",
+            testPath.toString()}, out));
 
     Set<Path> actualOnS3 = new HashSet<>();
     Set<Path> actualOnMS = new HashSet<>();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message