hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject hadoop git commit: HADOOP-14255. S3A to delete unnecessary fake directory objects in mkdirs(). Contributed by Mingliang Liu
Date Wed, 12 Apr 2017 21:31:02 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d77cada18 -> 508389ee5


HADOOP-14255. S3A to delete unnecessary fake directory objects in mkdirs(). Contributed by
Mingliang Liu

(cherry picked from commit 507845074970b90330db86bc1ff0124def550bb5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/508389ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/508389ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/508389ee

Branch: refs/heads/branch-2
Commit: 508389ee54091f4e2dbaa217cb71a06d0cc1cdb2
Parents: d77cada
Author: Mingliang Liu <liuml07@apache.org>
Authored: Thu Mar 30 13:03:34 2017 -0700
Committer: Mingliang Liu <liuml07@apache.org>
Committed: Wed Apr 12 14:30:54 2017 -0700

----------------------------------------------------------------------
 .../fs/contract/AbstractContractMkdirTest.java  | 42 ++++++++++++++++++++
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  3 +-
 2 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/508389ee/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
index 427b0e9..71d2706 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
@@ -126,4 +126,46 @@ public abstract class AbstractContractMkdirTest extends AbstractFSContractTestBa
     assertPathExists("check path existence without trailing slash failed",
         path("testmkdir/b"));
   }
+
+  @Test
+  public void testMkdirsPopulatingAllNonexistentAncestors() throws IOException {
+    describe("Verify mkdir will populate all its non-existent ancestors");
+    final FileSystem fs = getFileSystem();
+
+    final Path parent = path("testMkdirsPopulatingAllNonexistentAncestors");
+    assertTrue(fs.mkdirs(parent));
+    assertPathExists(parent + " should exist before making nested dir", parent);
+
+    Path nested = path(parent + "/a/b/c/d/e/f/g/h/i/j/k/L");
+    assertTrue(fs.mkdirs(nested));
+    while (nested != null && !nested.equals(parent) && !nested.isRoot())
{
+      assertPathExists(nested + " nested dir should exist", nested);
+      nested = nested.getParent();
+    }
+  }
+
+  @Test
+  public void testMkdirsDoesNotRemoveParentDirectories() throws IOException {
+    describe("Verify mkdir will make its parent existent");
+    final FileSystem fs = getFileSystem();
+
+    final Path parent = path("testMkdirsDoesNotRemoveParentDirectories");
+    assertTrue(fs.mkdirs(parent));
+
+    Path p = parent;
+    for (int i = 0; i < 10; i++) {
+      assertTrue(fs.mkdirs(p));
+      assertPathExists(p + " should exist after mkdir(" + p + ")", p);
+      p = path(p + "/dir-" + i);
+    }
+
+    // After mkdirs(sub-directory), its parent directory still exists
+    p = p.getParent();
+    while (p != null && !p.equals(parent) && !p.isRoot()) {
+      assertPathExists("Path " + p + " should exist", p);
+      assertIsDirectory(p);
+      p = p.getParent();
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/508389ee/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 07d9730c..f8cf272 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1524,8 +1524,6 @@ public class S3AFileSystem extends FileSystem {
    * @throws IOException other IO problems
    * @throws AmazonClientException on failures inside the AWS SDK
    */
-  // TODO: If we have created an empty file at /foo/bar and we then call
-  // mkdirs for /foo/bar/baz/roo what happens to the empty file /foo/bar/?
   private boolean innerMkdirs(Path f, FsPermission permission)
       throws IOException, FileAlreadyExistsException, AmazonClientException {
     LOG.debug("Making directory: {}", f);
@@ -1560,6 +1558,7 @@ public class S3AFileSystem extends FileSystem {
 
       String key = pathToKey(f);
       createFakeDirectory(key);
+      deleteUnnecessaryFakeDirectories(f.getParent());
       return true;
     }
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message