hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mackror...@apache.org
Subject [5/6] hadoop git commit: Revert "HADOOP-13760. S3Guard: add delete tracking."
Date Thu, 25 May 2017 12:13:17 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
deleted file mode 100644
index 43eb2c0..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-/**
- * Place for the S3A listing classes; keeps all the small classes under control.
- */
-public class TestListing extends AbstractS3AMockTest {
-
-  private static class MockRemoteIterator<FileStatus> implements
-      RemoteIterator<FileStatus> {
-    private Iterator<FileStatus> iterator;
-
-    MockRemoteIterator(Collection<FileStatus> source) {
-      iterator = source.iterator();
-    }
-
-    public boolean hasNext() {
-      return iterator.hasNext();
-    }
-
-    public FileStatus next() {
-      return iterator.next();
-    }
-  }
-
-  private FileStatus blankFileStatus(Path path) {
-    return new FileStatus(0, true, 0, 0, 0, path);
-  }
-
-  @Test
-  public void testTombstoneReconcilingIterator() throws Exception {
-    Path parent = new Path("/parent");
-    Path liveChild = new Path(parent, "/liveChild");
-    Path deletedChild = new Path(parent, "/deletedChild");
-    Path[] allFiles = {parent, liveChild, deletedChild};
-    Path[] liveFiles = {parent, liveChild};
-
-    Listing listing = new Listing(fs);
-    Collection<FileStatus> statuses = new ArrayList<>();
-    statuses.add(blankFileStatus(parent));
-    statuses.add(blankFileStatus(liveChild));
-    statuses.add(blankFileStatus(deletedChild));
-
-    Set<Path> tombstones = new HashSet<>();
-    tombstones.add(deletedChild);
-
-    RemoteIterator<FileStatus> sourceIterator = new MockRemoteIterator(
-        statuses);
-    RemoteIterator<LocatedFileStatus> locatedIterator =
-        listing.createLocatedFileStatusIterator(sourceIterator);
-    RemoteIterator<LocatedFileStatus> reconcilingIterator =
-        listing.createTombstoneReconcilingIterator(locatedIterator, tombstones);
-
-    Set<Path> expectedPaths = new HashSet<>();
-    expectedPaths.add(parent);
-    expectedPaths.add(liveChild);
-
-    Set<Path> actualPaths = new HashSet<>();
-    while (reconcilingIterator.hasNext()) {
-      actualPaths.add(reconcilingIterator.next().getPath());
-    }
-    Assert.assertTrue(actualPaths.equals(expectedPaths));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index dfa8a9e..99acf6e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.fs.s3a.s3guard;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.Tristate;
@@ -135,52 +134,12 @@ public abstract class MetadataStoreTestBase extends Assert {
   }
 
   /**
-   * Helper function for verifying DescendantsIterator and
-   * MetadataStoreListFilesIterator behavior.
-   * @param createNodes List of paths to create
-   * @param checkNodes List of paths that the iterator should return
-   * @throws IOException
-   */
-  private void doTestDescendantsIterator(
-      Class implementation, String[] createNodes,
-      String[] checkNodes) throws Exception {
-    // we set up the example file system tree in metadata store
-    for (String pathStr : createNodes) {
-      final FileStatus status = pathStr.contains("file")
-          ? basicFileStatus(strToPath(pathStr), 100, false)
-          : basicFileStatus(strToPath(pathStr), 0, true);
-      ms.put(new PathMetadata(status));
-    }
-
-    final PathMetadata rootMeta = new PathMetadata(makeDirStatus("/"));
-    RemoteIterator<FileStatus> iterator;
-    if (implementation == DescendantsIterator.class) {
-      iterator = new DescendantsIterator(ms, rootMeta);
-    } else if (implementation == MetadataStoreListFilesIterator.class) {
-      iterator = new MetadataStoreListFilesIterator(ms, rootMeta, false);
-    } else {
-      throw new UnsupportedOperationException("Unrecognized class");
-    }
-
-    final Set<String> actual = new HashSet<>();
-    while (iterator.hasNext()) {
-      final Path p = iterator.next().getPath();
-      actual.add(Path.getPathWithoutSchemeAndAuthority(p).toString());
-    }
-    LOG.info("We got {} by iterating DescendantsIterator", actual);
-
-    if (!allowMissing()) {
-      assertEquals(Sets.newHashSet(checkNodes), actual);
-    }
-  }
-
-  /**
    * Test that we can get the whole sub-tree by iterating DescendantsIterator.
    *
    * The tree is similar to or same as the example in code comment.
    */
   @Test
-  public void testDescendantsIterator() throws Exception {
+  public void testDescendantsIterator() throws IOException {
     final String[] tree = new String[] {
         "/dir1",
         "/dir1/dir2",
@@ -193,38 +152,26 @@ public abstract class MetadataStoreTestBase extends Assert {
         "/dir1/dir3/dir5/file4",
         "/dir1/dir3/dir6"
     };
-    doTestDescendantsIterator(DescendantsIterator.class,
-        tree, tree);
-  }
+    // we set up the example file system tree in metadata store
+    for (String pathStr : tree) {
+      final FileStatus status = pathStr.contains("file")
+          ? basicFileStatus(strToPath(pathStr), 100, false)
+          : basicFileStatus(strToPath(pathStr), 0, true);
+      ms.put(new PathMetadata(status));
+    }
 
-  /**
-   * Test that we can get the correct subset of the tree with
-   * MetadataStoreListFilesIterator.
-   *
-   * The tree is similar to or same as the example in code comment.
-   */
-  @Test
-  public void testMetadataStoreListFilesIterator() throws Exception {
-    final String[] wholeTree = new String[] {
-        "/dir1",
-        "/dir1/dir2",
-        "/dir1/dir3",
-        "/dir1/dir2/file1",
-        "/dir1/dir2/file2",
-        "/dir1/dir3/dir4",
-        "/dir1/dir3/dir5",
-        "/dir1/dir3/dir4/file3",
-        "/dir1/dir3/dir5/file4",
-        "/dir1/dir3/dir6"
-    };
-    final String[] leafNodes = new String[] {
-        "/dir1/dir2/file1",
-        "/dir1/dir2/file2",
-        "/dir1/dir3/dir4/file3",
-        "/dir1/dir3/dir5/file4"
-    };
-    doTestDescendantsIterator(MetadataStoreListFilesIterator.class, wholeTree,
-        leafNodes);
+    final Set<String> actual = new HashSet<>();
+    final PathMetadata rootMeta = new PathMetadata(makeDirStatus("/"));
+    for (DescendantsIterator desc = new DescendantsIterator(ms, rootMeta);
+         desc.hasNext();) {
+      final Path p = desc.next().getPath();
+      actual.add(Path.getPathWithoutSchemeAndAuthority(p).toString());
+    }
+    LOG.info("We got {} by iterating DescendantsIterator", actual);
+
+    if (!allowMissing()) {
+      assertEquals(Sets.newHashSet(tree), actual);
+    }
   }
 
   @Test
@@ -311,7 +258,7 @@ public abstract class MetadataStoreTestBase extends Assert {
     /* Ensure delete happened. */
     assertDirectorySize("/ADirectory1/db1", 1);
     PathMetadata meta = ms.get(strToPath("/ADirectory1/db1/file2"));
-    assertTrue("File deleted", meta == null || meta.isDeleted());
+    assertNull("File deleted", meta);
   }
 
   @Test
@@ -337,10 +284,10 @@ public abstract class MetadataStoreTestBase extends Assert {
     ms.deleteSubtree(strToPath(p + "/ADirectory1/db1/"));
 
     assertEmptyDirectory(p + "/ADirectory1");
-    assertDeleted(p + "/ADirectory1/db1");
-    assertDeleted(p + "/ADirectory1/file1");
-    assertDeleted(p + "/ADirectory1/file2");
-    assertDeleted(p + "/ADirectory1/db1/dc1/dd1/deepFile");
+    assertNotCached(p + "/ADirectory1/db1");
+    assertNotCached(p + "/ADirectory1/file1");
+    assertNotCached(p + "/ADirectory1/file2");
+    assertNotCached(p + "/ADirectory1/db1/dc1/dd1/deepFile");
     assertEmptyDirectory(p + "/ADirectory2");
   }
 
@@ -355,11 +302,11 @@ public abstract class MetadataStoreTestBase extends Assert {
     setUpDeleteTest();
 
     ms.deleteSubtree(strToPath("/"));
-    assertDeleted("/ADirectory1");
-    assertDeleted("/ADirectory2");
-    assertDeleted("/ADirectory2/db1");
-    assertDeleted("/ADirectory2/db1/file1");
-    assertDeleted("/ADirectory2/db1/file2");
+    assertNotCached("/ADirectory1");
+    assertNotCached("/ADirectory2");
+    assertNotCached("/ADirectory2/db1");
+    assertNotCached("/ADirectory2/db1/file1");
+    assertNotCached("/ADirectory2/db1/file2");
   }
 
   @Test
@@ -403,12 +350,6 @@ public abstract class MetadataStoreTestBase extends Assert {
       verifyFileStatus(meta.getFileStatus(), 100);
     }
 
-    if (!(ms instanceof NullMetadataStore)) {
-      ms.delete(strToPath(filePath));
-      meta = ms.get(strToPath(filePath));
-      assertTrue("Tombstone not left for deleted file", meta.isDeleted());
-    }
-
     meta = ms.get(strToPath(dirPath));
     if (!allowMissing() || meta != null) {
       assertNotNull("Get found file (dir)", meta);
@@ -500,7 +441,6 @@ public abstract class MetadataStoreTestBase extends Assert {
 
     dirMeta = ms.listChildren(strToPath("/a1"));
     if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
       assertListingsEqual(dirMeta.getListing(), "/a1/b1", "/a1/b2");
     }
 
@@ -546,7 +486,6 @@ public abstract class MetadataStoreTestBase extends Assert {
     Collection<PathMetadata> entries;
     DirListingMetadata dirMeta = ms.listChildren(strToPath("/"));
     if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
       assertNotNull("Listing root", dirMeta);
       entries = dirMeta.getListing();
       assertListingsEqual(entries, "/a1", "/a2", "/a3");
@@ -574,12 +513,13 @@ public abstract class MetadataStoreTestBase extends Assert {
     dirMeta = ms.listChildren(strToPath("/a1"));
     if (!allowMissing() || dirMeta != null) {
       assertNotNull("Listing /a1", dirMeta);
-      entries = dirMeta.withoutTombstones().getListing();
+      entries = dirMeta.getListing();
       assertListingsEqual(entries, "/a1/b2");
     }
 
     PathMetadata meta = ms.get(strToPath("/a1/b1/file1"));
-    assertTrue("Src path deleted", meta == null || meta.isDeleted());
+    // TODO allow return of PathMetadata with isDeleted == true
+    assertNull("Src path deleted", meta);
 
     // Assert dest looks right
     meta = ms.get(strToPath("/b1/file1"));
@@ -656,7 +596,7 @@ public abstract class MetadataStoreTestBase extends Assert {
     ms.prune(cutoff);
     ls = ms.listChildren(strToPath("/pruneFiles"));
     if (allowMissing()) {
-      assertDeleted("/pruneFiles/old");
+      assertNotCached("/pruneFiles/old");
     } else {
       assertListingsEqual(ls.getListing(), "/pruneFiles/new");
     }
@@ -685,7 +625,7 @@ public abstract class MetadataStoreTestBase extends Assert {
 
     ms.prune(cutoff);
 
-    assertDeleted("/pruneDirs/dir/file");
+    assertNotCached("/pruneDirs/dir/file");
   }
 
   /*
@@ -706,7 +646,6 @@ public abstract class MetadataStoreTestBase extends Assert {
         "file3"));
     DirListingMetadata dirMeta = ms.listChildren(strToPath(parent));
     if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
       assertNotNull("list after putListStatus", dirMeta);
       Collection<PathMetadata> entries = dirMeta.getListing();
       assertNotNull("listStatus has entries", entries);
@@ -761,7 +700,6 @@ public abstract class MetadataStoreTestBase extends Assert {
       assertNotNull("Directory " + pathStr + " in cache", dirMeta);
     }
     if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
       assertEquals("Number of entries in dir " + pathStr, size,
           nonDeleted(dirMeta.getListing()).size());
     }
@@ -770,27 +708,21 @@ public abstract class MetadataStoreTestBase extends Assert {
   /** @return only file statuses which are *not* marked deleted. */
   private Collection<PathMetadata> nonDeleted(
       Collection<PathMetadata> statuses) {
-    Collection<PathMetadata> currentStatuses = new ArrayList<>();
-    for (PathMetadata status : statuses) {
-      if (!status.isDeleted()) {
-        currentStatuses.add(status);
-      }
-    }
-    return currentStatuses;
+    /* TODO: filter out paths marked for deletion. */
+    return statuses;
   }
 
-  private void assertDeleted(String pathStr) throws IOException {
+  private void assertNotCached(String pathStr) throws IOException {
+    // TODO this should return an entry with deleted flag set
     Path path = strToPath(pathStr);
     PathMetadata meta = ms.get(path);
-    boolean cached = meta != null && !meta.isDeleted();
-    assertFalse(pathStr + " should not be cached.", cached);
+    assertNull(pathStr + " should not be cached.", meta);
   }
 
   protected void assertCached(String pathStr) throws IOException {
     Path path = strToPath(pathStr);
     PathMetadata meta = ms.get(path);
-    boolean cached = meta != null && !meta.isDeleted();
-    assertTrue(pathStr + " should be cached.", cached);
+    assertNotNull(pathStr + " should be cached.", meta);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index 27416bb..3584b54 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -294,7 +294,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
     if (oldMetas != null) {
       // put all metadata of old paths and verify
       ms.put(new DirListingMetadata(oldDir, oldMetas, false));
-      assertEquals(0, ms.listChildren(newDir).withoutTombstones().numEntries());
+      assertEquals(0, ms.listChildren(newDir).numEntries());
       assertTrue(CollectionUtils.isEqualCollection(oldMetas,
           ms.listChildren(oldDir).getListing()));
 
@@ -306,7 +306,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
 
     // move the old paths to new paths and verify
     ms.move(pathsToDelete, newMetas);
-    assertEquals(0, ms.listChildren(oldDir).withoutTombstones().numEntries());
+    assertEquals(0, ms.listChildren(oldDir).numEntries());
     if (newMetas != null) {
       assertTrue(CollectionUtils.isEqualCollection(newMetas,
           ms.listChildren(newDir).getListing()));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
index 89d0498..4cffc6f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -75,7 +75,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
 
   @Test
   public void testClearByAncestor() {
-    Map<Path, PathMetadata> map = new HashMap<>();
+    Map<Path, String> map = new HashMap<>();
 
     // 1. Test paths without scheme/host
     assertClearResult(map, "", "/", 0);
@@ -90,37 +90,21 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
     assertClearResult(map, p, "/invalid", 5);
   }
 
-  private static void populateMap(Map<Path, PathMetadata> map,
-      String prefix) {
-    populateEntry(map, new Path(prefix + "/dirA/dirB/"));
-    populateEntry(map, new Path(prefix + "/dirA/dirB/dirC"));
-    populateEntry(map, new Path(prefix + "/dirA/dirB/dirC/file1"));
-    populateEntry(map, new Path(prefix + "/dirA/dirB/dirC/file2"));
-    populateEntry(map, new Path(prefix + "/dirA/file1"));
+  private static void populateMap(Map<Path, String> map, String prefix) {
+    String dummyVal = "dummy";
+    map.put(new Path(prefix + "/dirA/dirB/"), dummyVal);
+    map.put(new Path(prefix + "/dirA/dirB/dirC"), dummyVal);
+    map.put(new Path(prefix + "/dirA/dirB/dirC/file1"), dummyVal);
+    map.put(new Path(prefix + "/dirA/dirB/dirC/file2"), dummyVal);
+    map.put(new Path(prefix + "/dirA/file1"), dummyVal);
   }
 
-  private static void populateEntry(Map<Path, PathMetadata> map,
-      Path path) {
-    map.put(path, new PathMetadata(new FileStatus(0, true, 0, 0, 0, path)));
-  }
-
-  private static int sizeOfMap(Map<Path, PathMetadata> map) {
-    int count = 0;
-    for (PathMetadata meta : map.values()) {
-      if (!meta.isDeleted()) {
-        count++;
-      }
-    }
-    return count;
-  }
-
-  private static void assertClearResult(Map <Path, PathMetadata> map,
+  private static void assertClearResult(Map <Path, String> map,
       String prefixStr, String pathStr, int leftoverSize) {
     populateMap(map, prefixStr);
-    LocalMetadataStore.deleteHashByAncestor(new Path(prefixStr + pathStr), map,
-        true);
+    LocalMetadataStore.clearHashByAncestor(new Path(prefixStr + pathStr), map);
     assertEquals(String.format("Map should have %d entries", leftoverSize),
-        leftoverSize, sizeOfMap(map));
+        leftoverSize, map.size());
     map.clear();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
index 876cc80..9b8e3c1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
@@ -114,7 +114,7 @@ public abstract class AbstractITestS3AMetadataStoreScale extends
         describe("Running move workload");
         NanoTimer moveTimer = new NanoTimer();
         LOG.info("Running {} moves of {} paths each", operations,
-            origMetas.size());
+            origPaths.size());
         for (int i = 0; i < operations; i++) {
           Collection<Path> toDelete;
           Collection<PathMetadata> toCreate;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message