hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject [15/21] hadoop git commit: MetadataStore.delete() improvements
Date Fri, 08 Jul 2016 21:30:22 GMT
MetadataStore.delete() improvements

- Add special case for root dir invalidation.  Add test case.

- Recursive delete on a sparsely-cached path should work,
  regardless of whether or not target path is cached.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0c1fee6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0c1fee6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0c1fee6

Branch: refs/heads/s3_create
Commit: e0c1fee6b7fc55b42f88348e0d0531b000d6f792
Parents: 1e48a8a
Author: Aaron Fabbri <fabbri@cloudera.com>
Authored: Tue Jul 5 15:14:01 2016 -0700
Committer: Aaron Fabbri <fabbri@cloudera.com>
Committed: Tue Jul 5 17:19:14 2016 -0700

----------------------------------------------------------------------
 .../hadoop/fs/s3a/LocalMetadataStore.java       | 18 +++++++++--------
 .../org/apache/hadoop/fs/s3a/MetadataStore.java |  1 +
 .../hadoop/fs/s3a/TestLocalMetadataStore.java   | 21 ++++++++++++++++++++
 3 files changed, 32 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0c1fee6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/LocalMetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/LocalMetadataStore.java
index a66f619..d5db39b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/LocalMetadataStore.java
@@ -50,23 +50,23 @@ public class LocalMetadataStore implements MetadataStore {
 
     // Delete entry from file cache, then from cached parent directory, if any
 
+    CachedFileStatus cfs = null;
     synchronized (this) {
       // Remove target file/dir
-      CachedFileStatus cfs = fileHash.remove(f);
-
-      if (cfs == null)
-        return false;
+      cfs = fileHash.remove(f);
 
       // Update parent dir listing, if any
-      dirHashDeleteFile(cfs.getFileStatus().getPath());
+      dirHashDeleteFile(f);
 
-      if (recursive && cfs.getFileStatus().isDirectory()) {
+      if (recursive) {
         // Remove all entries that have this dir as path prefix.
         clearHashByAncestor(f, dirHash);
         clearHashByAncestor(f, fileHash);
       }
-      return true;
     }
+    // XXX TODO this is a meaningless return value.  Consider changing
+    // interface to return void?
+    return (cfs != null);
   }
 
   @Nullable
@@ -150,7 +150,9 @@ public class LocalMetadataStore implements MetadataStore {
    * All paths here are absolute.  Dir does not count as its own ancestor.
    */
   private static boolean isAncestorOf(Path ancestor, Path f) {
-    String aStr = ancestor.toString() + "/";
+    String aStr = ancestor.toString();
+    if (! ancestor.isRoot())
+      aStr += "/";
     String fStr = f.toString();
     return (fStr.indexOf(aStr) == 0);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0c1fee6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
index a24cf76..d6ab086 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataStore.java
@@ -36,6 +36,7 @@ import java.io.IOException;
 @InterfaceStability.Unstable
 public interface MetadataStore {
 
+  // What does return value mean?
   boolean delete(Path f, boolean recursive) throws IOException;
 
   @Nullable

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0c1fee6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
index 4af60ea..e3f93c3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
@@ -79,6 +79,15 @@ public class TestLocalMetadataStore extends Assert {
   }
 
   @Test
+  public void testRootDir() throws Exception {
+    Path f = new Path("/");
+
+    // XXX TODO quiet logging
+    LOG.info("\"{}\" is{} root", f, f.isRoot() ? "" : " not");
+
+  }
+
+  @Test
   public void testDelete() throws Exception {
     setUpDeleteTest();
 
@@ -106,6 +115,18 @@ public class TestLocalMetadataStore extends Assert {
     assertEmptyDirectory("/ADirectory2");
   }
 
+  @Test
+  public void testDeleteRecursiveRoot() throws Exception {
+    setUpDeleteTest();
+
+    lms.delete(new Path("/"), true);
+    assertNotCached("/ADirectory1");
+    assertNotCached("/ADirectory2");
+    assertNotCached("/ADirectory2/db1");
+    assertNotCached("/ADirectory2/db1/file1");
+    assertNotCached("/ADirectory2/db1/file2");
+  }
+
   private void setUpDeleteTest() throws IOException {
     createNewDirs("/ADirectory1", "/ADirectory2", "/ADirectory1/db1");
     lms.putNew(new CachedFileStatus(makeFileStatus("/ADirectory1/db1/file1", 100)));


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message