hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject [07/21] hadoop git commit: More LocalMetadataStore unit tests, get() and listStatus()
Date Fri, 08 Jul 2016 21:30:14 GMT
More LocalMetadataStore unit tests, get() and listStatus()


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e20b3af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e20b3af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e20b3af

Branch: refs/heads/s3_create
Commit: 0e20b3af7bedc39c2e19c377d4f0d65457871e13
Parents: 4f9b84d
Author: Aaron Fabbri <fabbri@cloudera.com>
Authored: Wed Jun 29 17:46:35 2016 -0700
Committer: Aaron Fabbri <fabbri@cloudera.com>
Committed: Wed Jun 29 17:46:35 2016 -0700

----------------------------------------------------------------------
 .../hadoop/fs/s3a/TestLocalMetadataStore.java   | 57 ++++++++++++++++++++
 1 file changed, 57 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e20b3af/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
index 6accd20..08a5c97 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestLocalMetadataStore.java
@@ -14,8 +14,10 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Unit tests for LocalMetadataStore.
@@ -116,12 +118,54 @@ public class TestLocalMetadataStore extends Assert {
 
   @Test
   public void testGet() throws Exception {
+    final String filePath = "/a1/b1/c1/some_file";
+    final String dirPath = "/a1/b1/c1/d1";
+    lms.put(new CachedFileStatus(makeFileStatus(filePath, 100)));
+    lms.put(new CachedFileStatus(makeDirStatus(dirPath)));
+    CachedFileStatus cfs = lms.get(new Path(filePath));
+    assertNotNull("Get found file", cfs);
+    assertEquals("File size as expected", cfs.getFileStatus().getLen(), 100);
+    assertFalse("Found file, not dir", cfs.getFileStatus().isDirectory());
 
+    cfs = lms.get(new Path(dirPath));
+    assertNotNull("Get found file (dir)", cfs);
+    assertTrue("Found dir", cfs.getFileStatus().isDirectory());
+
+    cfs = lms.get(new Path("bollocks"));
+    assertNull("Don't get non-existent file", cfs);
   }
 
 
   @Test
   public void testListStatus() throws Exception {
+    createNewDirs("/a1","/a2", "/a1/b1", "/a1/b2", "/a1/b1/c1", "/a1/b1/c1/d1");
+    lms.putNew(new CachedFileStatus(makeFileStatus("/a1/b1/file1", 100)));
+    lms.putNew(new CachedFileStatus(makeFileStatus("/a1/b1/file2", 100)));
+
+    CachedDirectory cd = lms.listStatus(new Path("/"));
+    if (cd != null) {
+      /* Cache has no way of knowing it has all entries for root unless we
+       * specifically tell it via putListStatus() with
+       * CachedDirectory.isFullyCached = true */
+      assertFalse("Root dir is not cached, or partially cached", cd.isFullyCached());
+      assertFileStatusesEqual(cd.getFileStatuses(), "/a1", "/a2");
+    }
+
+    cd = lms.listStatus(new Path("/a1"));
+    assertFileStatusesEqual(cd.getFileStatuses(), "/a1/b1", "/a1/b2");
+    /* When the cache sees a directory creation, it starts in state fully cached
+     * and stays that way unless invalidated.  Assumption is that it sees all
+     * modifications to the directory via putNew()/delete()
+     */
+    assertTrue("Created dir is fully cached", cd.isFullyCached());
+
+    cd = lms.listStatus(new Path("/a1/b1"));
+    assertFileStatusesEqual(cd.getFileStatuses(), "/a1/b1/file1",
+      "/a1/b1/file2", "/a1/b1/c1");
+  }
+
+  @Test
+  public void testInvalidListStatus() throws Exception {
 
   }
 
@@ -142,6 +186,19 @@ public class TestLocalMetadataStore extends Assert {
     assertNull("e1 deleted", testMap.get("/a1/b1/c1/d1/e1"));
   }
 
+  private void assertFileStatusesEqual(FileStatus[] statuses, String ...pathStrs) {
+    Set<Path> a = new HashSet<>();
+    for (FileStatus fs : statuses) {
+      a.add(fs.getPath());
+    }
+
+    Set<Path> b = new HashSet<>();
+    for (String ps : pathStrs) {
+      b.add(new Path(ps));
+    }
+    assertTrue("Same set of files", a.equals(b));
+  }
+
   private void createNewDirs(String... dirs)
       throws IOException {
     for (String pathStr : dirs) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message