Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: mackrorysd@apache.org
To: common-commits@hadoop.apache.org
Message-Id: <7c9051cae92a410fba82911cfae4d473@git.apache.org>
Subject: hadoop git commit: HADOOP-14527. S3Guard: ITestS3GuardListConsistency
 is too slow. Contributed by Aaron Fabbri.
Date: Fri, 23 Jun 2017 20:54:45 +0000 (UTC)
archived-at: Fri, 23 Jun 2017 20:54:47 -0000

Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 0db7176ba -> be922ba6a


HADOOP-14527. S3Guard: ITestS3GuardListConsistency is too slow.
Contributed by Aaron Fabbri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be922ba6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be922ba6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be922ba6

Branch: refs/heads/HADOOP-13345
Commit: be922ba6ad4d4eb68ae1df437e578e361bc3ee37
Parents: 0db7176
Author: Sean Mackrory <mackrorysd@apache.org>
Authored: Fri Jun 23 14:40:45 2017 -0600
Committer: Sean Mackrory <mackrorysd@apache.org>
Committed: Fri Jun 23 14:41:18 2017 -0600

----------------------------------------------------------------------
 .../fs/s3a/InconsistentAmazonS3Client.java      | 29 ++++++++++++++++++++
 .../fs/s3a/ITestS3GuardListConsistency.java     | 27 ++++++++++--------
 2 files changed, 45 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be922ba6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
index 85f4a2f..ba40d34 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
@@ -22,6 +22,7 @@ import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.DeleteObjectRequest;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
@@ -137,6 +138,34 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
         delayKeyMsec, delayKeySubstring, delayKeyProbability);
   }
 
+  /**
+   * Clear all oustanding inconsistent keys.  After calling this function,
+   * listings should behave normally (no failure injection), until additional
+   * keys are matched for delay, e.g. via putObject(), deleteObject().
+   */
+  public void clearInconsistency() {
+    LOG.info("clearing all delayed puts / deletes");
+    delayedDeletes.clear();
+    delayedPutKeys.clear();
+  }
+
+  /**
+   * Convenience function for test code to cast from supertype.
+   * @param c supertype to cast from
+   * @return subtype, not null
+   * @throws Exception on error
+   */
+  public static InconsistentAmazonS3Client castFrom(AmazonS3 c) throws
+      Exception {
+    InconsistentAmazonS3Client ic = null;
+    if (c instanceof InconsistentAmazonS3Client) {
+      ic = (InconsistentAmazonS3Client) c;
+    }
+    Preconditions.checkNotNull(ic, "Not an instance of " +
+        "InconsistentAmazonS3Client");
+    return ic;
+  }
+
   @Override
   public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
       deleteObjectsRequest) throws AmazonClientException,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be922ba6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
index b0da172..6cff533 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs.s3a;
 
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.AmazonS3;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -83,7 +82,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
       for (Path mkdir : mkdirs) {
         assertTrue(fs.mkdirs(mkdir));
       }
-      Thread.sleep(DEFAULT_DELAY_KEY_MSEC);
+      clearInconsistency(fs);
     }
 
     assertTrue("srcdirs and dstdirs must have equal length",
@@ -176,7 +175,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
     for (Path path : testDirs) {
       assertTrue(fs.mkdirs(path));
     }
-    Thread.sleep(2 * DEFAULT_DELAY_KEY_MSEC);
+    clearInconsistency(fs);
     for (Path path : testDirs) {
       assertTrue(fs.delete(path, false));
     }
@@ -216,7 +215,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
     for (Path path : testDirs) {
       assertTrue(fs.mkdirs(path));
     }
-    Thread.sleep(2 * DEFAULT_DELAY_KEY_MSEC);
+    clearInconsistency(fs);
     assertTrue(fs.delete(testDirs[1], false));
     assertTrue(fs.delete(testDirs[2], false));
 
@@ -229,8 +228,8 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
     assertTrue(list.contains(path("a3/b/dir1")));
     assertFalse(list.contains(path("a3/b/dir2")));
     // This should fail without S3Guard, and succeed with it.
-    assertFalse(list.contains(
-        path("a3/b/dir3-" + DEFAULT_DELAY_KEY_SUBSTRING)));
+    assertFalse(list.contains(path("a3/b/dir3-" +
+        DEFAULT_DELAY_KEY_SUBSTRING)));
 
     try {
       RemoteIterator<LocatedFileStatus> old = fs.listFilesAndEmptyDirectories(
@@ -287,9 +286,9 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
     String rootDir = "doTestConsistentListLocatedStatusAfterPut";
     fs.mkdirs(path(rootDir));
 
-    final int[] numOfPaths = {0, 1, 10};
+    final int[] numOfPaths = {0, 1, 5};
     for (int normalPathNum : numOfPaths) {
-      for (int delayedPathNum : numOfPaths) {
+      for (int delayedPathNum : new int[] {0, 2}) {
         LOG.info("Testing with normalPathNum={}, delayedPathNum={}",
             normalPathNum, delayedPathNum);
         doTestConsistentListLocatedStatusAfterPut(fs, rootDir, normalPathNum,
@@ -352,10 +351,10 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
     // skip it.
     Assume.assumeTrue(fs.hasMetadataStore());
 
-    final int[] numOfPaths = {0, 1, 2};
+    final int[] numOfPaths = {0, 2};
     for (int dirNum : numOfPaths) {
       for (int normalFile : numOfPaths) {
-        for (int delayedFile : numOfPaths) {
+        for (int delayedFile : new int[] {0, 1}) {
           for (boolean recursive : new boolean[] {true, false}) {
             doTestListFiles(fs, dirNum, normalFile, delayedFile, recursive);
           }
@@ -498,7 +497,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
         touch(fs, new Path(new Path(root, "dir" + i), "file" + i + "-" + j));
       }
     }
-    Thread.sleep(2 * DEFAULT_DELAY_KEY_MSEC);
+    clearInconsistency(fs);
 
     AmazonS3 client = fs.getAmazonS3Client();
     String key = fs.pathToKey(root) + "/";
@@ -536,4 +535,10 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
         postDeleteUndelimited.getCommonPrefixes().size()
     );
   }
+
+  private static void clearInconsistency(S3AFileSystem fs) throws Exception {
+    AmazonS3 s3 = fs.getAmazonS3Client();
+    InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3);
+    ic.clearInconsistency();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org