hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject hadoop git commit: HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse.
Date Sun, 03 Apr 2016 15:41:05 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 aa3874899 -> 20e808d83


HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list.
author: Pieter Reuse.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20e808d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20e808d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20e808d8

Branch: refs/heads/branch-2
Commit: 20e808d83e7dc5a49f629e4031dbceb73f9b4cb8
Parents: aa38748
Author: Steve Loughran <stevel@apache.org>
Authored: Sun Apr 3 16:39:14 2016 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Sun Apr 3 16:39:51 2016 +0100

----------------------------------------------------------------------
 .../AbstractContractGetFileStatusTest.java      | 23 ++++++++++++++++++++
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  7 ++++--
 .../src/test/resources/contract/s3a.xml         |  5 +++++
 3 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e808d8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
index 7ed375e..3e5bb12 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
@@ -19,10 +19,14 @@
 package org.apache.hadoop.fs.contract;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
 
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -58,4 +62,23 @@ public abstract class AbstractContractGetFileStatusTest extends
       handleExpectedException(e);
     }
   }
+
+  @Test
+  public void testListStatusEmptyDirectory() throws IOException {
+    // remove the test directory
+    FileSystem fs = getFileSystem();
+    assertTrue(fs.delete(getContract().getTestPath(), true));
+
+    // create a - non-qualified - Path for a subdir
+    Path subfolder = getContract().getTestPath().suffix("/"+testPath.getName());
+    assertTrue(fs.mkdirs(subfolder));
+
+    // assert empty ls on the empty dir
+    assertEquals("ls on an empty directory not of length 0", 0,
+        fs.listStatus(subfolder).length);
+
+    // assert non-empty ls on parent dir
+    assertTrue("ls on a non-empty directory of length 0",
+        fs.listStatus(getContract().getTestPath()).length > 0);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e808d8/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 33db86e..23d17fb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -846,11 +846,14 @@ public class S3AFileSystem extends FileSystem {
       ObjectListing objects = s3.listObjects(request);
       statistics.incrementReadOps(1);
 
+      Path fQualified = f.makeQualified(uri, workingDir);
+
       while (true) {
         for (S3ObjectSummary summary : objects.getObjectSummaries()) {
           Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
           // Skip over keys that are ourselves and old S3N _$folder$ files
-          if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
+          if (keyPath.equals(fQualified) ||
+              summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
             if (LOG.isDebugEnabled()) {
               LOG.debug("Ignoring: " + keyPath);
             }
@@ -865,7 +868,7 @@ public class S3AFileSystem extends FileSystem {
           } else {
             result.add(new S3AFileStatus(summary.getSize(),
                 dateToLong(summary.getLastModified()), keyPath,
-                getDefaultBlockSize(f.makeQualified(uri, workingDir))));
+                getDefaultBlockSize(fQualified)));
             if (LOG.isDebugEnabled()) {
               LOG.debug("Adding: fi: " + keyPath);
             }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e808d8/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
index 4f9c081..be1e7ca 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
@@ -78,6 +78,11 @@
   </property>
 
   <property>
+    <name>fs.contract.supports-getfilestatus</name>
+    <value>true</value>
+  </property>
+
+  <property>
     <name>fs.contract.supports-seek</name>
     <value>true</value>
   </property>


Mime
View raw message