hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject hadoop git commit: HADOOP-14036. S3Guard: intermittent duplicate item keys failure. Contributed by Mingliang Liu
Date Fri, 17 Mar 2017 18:00:34 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 322a2bc74 -> b54e1b229


HADOOP-14036. S3Guard: intermittent duplicate item keys failure. Contributed by Mingliang
Liu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b54e1b22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b54e1b22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b54e1b22

Branch: refs/heads/HADOOP-13345
Commit: b54e1b2298bf5aeef38f5db6a232199dd961aa0e
Parents: 322a2bc
Author: Mingliang Liu <liuml07@apache.org>
Authored: Wed Mar 15 10:56:02 2017 -0700
Committer: Mingliang Liu <liuml07@apache.org>
Committed: Fri Mar 17 11:00:18 2017 -0700

----------------------------------------------------------------------
 .../hadoop/fs/FileSystemContractBaseTest.java       | 16 +++++++++++++++-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java     | 10 ++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54e1b22/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 233e524..0274765 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -669,12 +669,26 @@ public abstract class FileSystemContractBaseTest extends TestCase {
 
   /**
    * This a sanity check to make sure that any filesystem's handling of
-   * renames doesn't cause any regressions
+   * renames empty dirs doesn't cause any regressions.
+   */
+  public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable {
+    if (!renameSupported()) return;
+    Path parentdir = path("test/parentdir");
+    fs.mkdirs(parentdir);
+    Path dest = path("test/parentdirdest");
+    rename(parentdir, dest, true, false, true);
+  }
+
+  /**
+   * This a sanity check to make sure that any filesystem's handling of
+   * renames non-empty dirs doesn't cause any regressions.
    */
   public void testRenameToDirWithSamePrefixAllowed() throws Throwable {
     if (!renameSupported()) return;
     Path parentdir = path("test/parentdir");
     fs.mkdirs(parentdir);
+    // Before renaming, we create one file under the source parent directory
+    createFile(new Path(parentdir, "mychild"));
     Path dest = path("test/parentdirdest");
     rename(parentdir, dest, true, false, true);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54e1b22/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index baa26a7..dcdd3b6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -918,8 +918,14 @@ public class S3AFileSystem extends FileSystem {
         }
       }
 
-      // We moved all the children, now move the top-level dir.
-      S3Guard.addMoveDir(metadataStore, srcPaths, dstMetas, src, dst, username);
+      // We moved all the children, now move the top-level dir if it's non-empty
+      // Empty directory has been added as the object summary of listObjects
+      if (srcStatus.isEmptyDirectory() == Tristate.FALSE) {
+        LOG.debug("To move the non-empty top-level dir src={} and dst={}",
+            src, dst);
+        S3Guard.addMoveDir(metadataStore, srcPaths, dstMetas, src, dst,
+            username);
+      }
     }
 
     metadataStore.move(srcPaths, dstMetas);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message