Return-Path: <common-commits-return-49762-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org>
X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io
Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io
Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183])
	by cust-asf2.ponee.io (Postfix) with ESMTP id 284A3200828
	for <archive-asf-public-internal@cust-asf2.ponee.io>; Fri, 13 May 2016 20:00:52 +0200 (CEST)
Received: by cust-asf.ponee.io (Postfix)
	id 26F02160A12; Fri, 13 May 2016 18:00:52 +0000 (UTC)
Delivered-To: archive-asf-public@cust-asf.ponee.io
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by cust-asf.ponee.io (Postfix) with SMTP id EB5181602BE
	for <archive-asf-public@cust-asf.ponee.io>; Fri, 13 May 2016 20:00:50 +0200 (CEST)
Received: (qmail 16626 invoked by uid 500); 13 May 2016 18:00:40 -0000
Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:common-commits-help@hadoop.apache.org>
List-Unsubscribe: <mailto:common-commits-unsubscribe@hadoop.apache.org>
List-Post: <mailto:common-commits@hadoop.apache.org>
List-Id: <common-commits.hadoop.apache.org>
Delivered-To: mailing list common-commits@hadoop.apache.org
Received: (qmail 14815 invoked by uid 99); 13 May 2016 18:00:39 -0000
Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23)
    by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 May 2016 18:00:39 +0000
Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33)
	id 5719FEABDC; Fri, 13 May 2016 18:00:39 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: aengineer@apache.org
To: common-commits@hadoop.apache.org
Date: Fri, 13 May 2016 18:01:15 -0000
Message-Id: <030846136c1b4e0ea78a683768c1a101@git.apache.org>
In-Reply-To: <f6f2442474ac467faf2a6939df5b520b@git.apache.org>
References: <f6f2442474ac467faf2a6939df5b520b@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: [38/41] hadoop git commit: HADOOP-13113 Enable parallel test
 execution for hadoop-aws. Chris Nauroth via stevel
archived-at: Fri, 13 May 2016 18:00:52 -0000

HADOOP-13113 Enable parallel test execution for hadoop-aws. Chris Nauroth via stevel


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd3a8bed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd3a8bed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd3a8bed

Branch: refs/heads/HDFS-1312
Commit: dd3a8bed0aa183d5dbb523742eafd6f283cfe479
Parents: fa440a3
Author: Steve Loughran <stevel@apache.org>
Authored: Fri May 13 10:46:15 2016 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Fri May 13 10:47:12 2016 +0100

----------------------------------------------------------------------
 hadoop-tools/hadoop-aws/pom.xml                 | 116 ++++++++++++++++---
 .../src/site/markdown/tools/hadoop-aws/index.md |  22 ++++
 .../hadoop/fs/contract/s3/S3Contract.java       |  10 ++
 .../hadoop/fs/contract/s3a/S3AContract.java     |   9 +-
 .../fs/contract/s3a/TestS3AContractRootDir.java |  39 ++++++-
 .../fs/contract/s3n/NativeS3Contract.java       |   9 +-
 .../hadoop/fs/s3a/scale/S3AScaleTestBase.java   |   4 +-
 7 files changed, 192 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd3a8bed/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 242cd2f..dfcb1b0 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -34,6 +34,7 @@
   <properties>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
+    <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
   </properties>
 
   <profiles>
@@ -59,7 +60,109 @@
         <maven.test.skip>false</maven.test.skip>
       </properties>
     </profile>
+    <profile>
+      <id>parallel-tests</id>
+      <build>
+        <plugins>
+          <plugin>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>create-parallel-tests-dirs</id>
+                <phase>test-compile</phase>
+                <configuration>
+                  <target>
+                    <script language="javascript"><![CDATA[
+                      var baseDirs = [
+                          "${test.build.data}",
+                          "${test.build.dir}",
+                          "${hadoop.tmp.dir}" ];
+                      for (var i in baseDirs) {
+                        for (var j = 1; j <= ${testsThreadCount}; ++j) {
+                          var mkdir = project.createTask("mkdir");
+                          mkdir.setDir(new java.io.File(baseDirs[i], j));
+                          mkdir.perform();
+                        }
+                      }
+                    ]]></script>
+                  </target>
+                </configuration>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>default-test</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>test</goal>
+                </goals>
+                <configuration>
+                  <forkCount>${testsThreadCount}</forkCount>
+                  <reuseForks>false</reuseForks>
+                  <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+                  <systemPropertyVariables>
+                    <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+                    <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+                    <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
 
+                    <!-- Due to a Maven quirk, setting this to just -->
+                    <!-- surefire.forkNumber won't do the parameter -->
+                    <!-- substitution.  Putting a prefix in front of it like -->
+                    <!-- "fork-" makes it work. -->
+                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                  </systemPropertyVariables>
+                  <!-- Some tests cannot run in parallel.  Tests that cover -->
+                  <!-- access to the root directory must run in isolation -->
+                  <!-- from anything else that could modify the bucket. -->
+                  <!-- S3A tests that cover multi-part upload must run in -->
+                  <!-- isolation, because the file system is configured to -->
+                  <!-- purge existing multi-part upload data on -->
+                  <!-- initialization.  MiniYARNCluster has not yet been -->
+                  <!-- changed to handle parallel test execution gracefully. -->
+                  <!-- Exclude all of these tests from parallel execution, -->
+                  <!-- and instead run them sequentially in a separate -->
+                  <!-- Surefire execution step later. -->
+                  <excludes>
+                    <exclude>**/TestJets3tNativeS3FileSystemContract.java</exclude>
+                    <exclude>**/TestS3ABlockingThreadPool.java</exclude>
+                    <exclude>**/TestS3AFastOutputStream.java</exclude>
+                    <exclude>**/TestS3AFileSystemContract.java</exclude>
+                    <exclude>**/TestS3AMiniYarnCluster.java</exclude>
+                    <exclude>**/Test*Root*.java</exclude>
+                  </excludes>
+                </configuration>
+              </execution>
+              <execution>
+                <id>sequential-tests</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>test</goal>
+                </goals>
+                <configuration>
+                  <!-- Do a sequential run for tests that cannot handle -->
+                  <!-- parallel execution. -->
+                  <includes>
+                    <include>**/TestJets3tNativeS3FileSystemContract.java</include>
+                    <include>**/TestS3ABlockingThreadPool.java</include>
+                    <include>**/TestS3AFastOutputStream.java</include>
+                    <include>**/TestS3AFileSystemContract.java</include>
+                    <include>**/TestS3AMiniYarnCluster.java</include>
+                    <include>**/Test*Root*.java</include>
+                  </includes>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <build>
@@ -116,30 +219,17 @@
       <artifactId>hadoop-common</artifactId>
       <scope>compile</scope>
     </dependency>
-
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
-
-    <!-- see ../../hadoop-project/pom.xml for versions -->
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-databind</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-annotations</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>
       <scope>compile</scope>
     </dependency>
-
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd3a8bed/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 2259200..95e3274 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -730,3 +730,25 @@ or in batch runs.
 
 Smaller values should result in faster test runs, especially when the object
 store is a long way away.
+
+### Running the Tests
+
+After completing the configuration, execute the test run through Maven.
+
+    mvn clean test
+
+It's also possible to execute multiple test suites in parallel by enabling the
+`parallel-tests` Maven profile.  The tests spend most of their time blocked on
+network I/O with the S3 service, so running in parallel tends to complete full
+test runs faster.
+
+    mvn -Pparallel-tests clean test
+
+Some tests must run with exclusive access to the S3 bucket, so even with the
+`parallel-tests` profile enabled, several test suites will run in serial in a
+separate Maven execution step after the parallel tests.
+
+By default, the `parallel-tests` profile runs 4 test suites concurrently.  This
+can be tuned by passing the `testsThreadCount` argument.
+
+    mvn -Pparallel-tests -DtestsThreadCount=8 clean test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd3a8bed/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
index b388ce7..483b78c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
@@ -19,8 +19,12 @@
 package org.apache.hadoop.fs.contract.s3;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
 
+/**
+ * The contract of S3: only enabled if the test bucket is provided.
+ */
 public class S3Contract extends AbstractBondedFSContract {
 
   public static final String CONTRACT_XML = "contract/s3.xml";
@@ -37,4 +41,10 @@ public class S3Contract extends AbstractBondedFSContract {
     return "s3";
   }
 
+  @Override
+  public Path getTestPath() {
+    String testUniqueForkId = System.getProperty("test.unique.fork.id");
+    return testUniqueForkId == null ? super.getTestPath() :
+        new Path("/" + testUniqueForkId, "test");
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd3a8bed/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
index cbdb3bd..e9024b5 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
@@ -19,10 +19,11 @@
 package org.apache.hadoop.fs.contract.s3a;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
 
 /**
- * The contract of S3A: only enabled if the test bucket is provided
+ * The contract of S3A: only enabled if the test bucket is provided.
  */
 public class S3AContract extends AbstractBondedFSContract {
 
@@ -40,4 +41,10 @@ public class S3AContract extends AbstractBondedFSContract {
     return "s3a";
   }
 
+  @Override
+  public Path getTestPath() {
+    String testUniqueForkId = System.getProperty("test.unique.fork.id");
+    return testUniqueForkId == null ? super.getTestPath() :
+        new Path("/" + testUniqueForkId, "test");
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd3a8bed/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
index 5e2352c..1cf1eb8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
@@ -18,18 +18,55 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
- * root dir operations against an S3 bucket
+ * root dir operations against an S3 bucket.
  */
 public class TestS3AContractRootDir extends
     AbstractContractRootDirectoryTest {
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestS3AContractRootDir.class);
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
   }
+
+  @Override
+  public void testListEmptyRootDirectory() throws IOException {
+    for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
+      try {
+        super.testListEmptyRootDirectory();
+        break;
+      } catch (AssertionError | FileNotFoundException e) {
+        if (attempt < maxAttempts) {
+          LOG.info("Attempt {} of {} for empty root directory test failed.  "
+              + "This is likely caused by eventual consistency of S3 "
+              + "listings.  Attempting retry.", attempt, maxAttempts);
+          try {
+            Thread.sleep(1000);
+          } catch (InterruptedException e2) {
+            Thread.currentThread().interrupt();
+            fail("Test interrupted.");
+            break;
+          }
+        } else {
+          LOG.error(
+              "Empty root directory test failed {} attempts.  Failing test.",
+              maxAttempts);
+          throw e;
+        }
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd3a8bed/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
index ace6444..5796d88 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
@@ -19,10 +19,11 @@
 package org.apache.hadoop.fs.contract.s3n;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
 
 /**
- * The contract of S3N: only enabled if the test bucket is provided
+ * The contract of S3N: only enabled if the test bucket is provided.
  */
 public class NativeS3Contract extends AbstractBondedFSContract {
 
@@ -40,4 +41,10 @@ public class NativeS3Contract extends AbstractBondedFSContract {
     return "s3n";
   }
 
+  @Override
+  public Path getTestPath() {
+    String testUniqueForkId = System.getProperty("test.unique.fork.id");
+    return testUniqueForkId == null ? super.getTestPath() :
+        new Path("/" + testUniqueForkId, "test");
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd3a8bed/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
index 42c552a..20ef6ec 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
@@ -127,7 +127,9 @@ public class S3AScaleTestBase extends Assert {
   }
 
   protected Path getTestPath() {
-    return new Path("/tests3a");
+    String testUniqueForkId = System.getProperty("test.unique.fork.id");
+    return testUniqueForkId == null ? new Path("/tests3a") :
+        new Path("/" + testUniqueForkId, "tests3a");
   }
 
   protected long getOperationCount() {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org