Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: common-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1515955 - in
 /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
 src/main/java/org/apache/hadoop/fs/ src/test/java/org/apache/hadoop/fs/
Date: Tue, 20 Aug 2013 20:19:08 -0000
To: common-commits@hadoop.apache.org
From: wang@apache.org
Message-Id: <20130820201908.742EB2388999@eris.apache.org>

Author: wang
Date: Tue Aug 20 20:19:07 2013
New Revision: 1515955

URL: http://svn.apache.org/r1515955
Log:
HADOOP-9877. Fix listing of snapshot directories in globStatus. (Binglin Chang via Andrew Wang)

Modified:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Aug 20 20:19:07 2013
@@ -349,6 +349,8 @@ Release 2.3.0 - UNRELEASED
     HADOOP-9865.  FileContext#globStatus has a regression with respect to
     relative path.  (Chuan Lin via Colin Patrick McCabe)
 
+    HADOOP-9877. Fix listing of snapshot directories in globStatus.
+    (Binglin Chang via Andrew Wang)
 
 Release 2.1.1-beta - UNRELEASED
 

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java Tue Aug 20 20:19:07 2013
@@ -62,6 +62,18 @@ class Globber {
     }
   }
 
+  private FileStatus getFileLinkStatus(Path path) {
+    try {
+      if (fs != null) {
+        return fs.getFileLinkStatus(path);
+      } else {
+        return fc.getFileLinkStatus(path);
+      }
+    } catch (IOException e) {
+      return null;
+    }
+  }
+
   private FileStatus[] listStatus(Path path) {
     try {
       if (fs != null) {
@@ -122,6 +134,18 @@ class Globber {
     return authority ;
   }
 
+  /**
+   * The glob filter builds a regexp per path component.  If the component
+   * does not contain a shell metachar, then it falls back to appending the
+   * raw string to the list of built up paths.  This raw path needs to have
+   * the quoting removed.  Ie. convert all occurrences of "\X" to "X"
+   * @param name of the path component
+   * @return the unquoted path component
+   */
+  private static String unquotePathComponent(String name) {
+    return name.replaceAll("\\\\(.)", "$1");
+  }
+
   public FileStatus[] glob() throws IOException {
     // First we get the scheme and authority of the pattern that was passed
     // in.
@@ -176,14 +200,30 @@ class Globber {
               resolvedCandidate.isDirectory() == false) {
             continue;
           }
-          FileStatus[] children = listStatus(candidate.getPath());
-          for (FileStatus child : children) {
-            // Set the child path based on the parent path.
-            // This keeps the symlinks in our path.
-            child.setPath(new Path(candidate.getPath(),
-                    child.getPath().getName()));
-            if (globFilter.accept(child.getPath())) {
-              newCandidates.add(child);
+          // For components without pattern, we get its FileStatus directly
+          // using getFileLinkStatus for two reasons:
+          // 1. It should be faster to only get FileStatus needed rather than
+          //    get all children.
+          // 2. Some special filesystem directories (e.g. HDFS snapshot
+          //    directories) are not returned by listStatus, but do exist if
+          //    checked explicitly via getFileLinkStatus.
+          if (globFilter.hasPattern()) {
+            FileStatus[] children = listStatus(candidate.getPath());
+            for (FileStatus child : children) {
+              // Set the child path based on the parent path.
+              // This keeps the symlinks in our path.
+              child.setPath(new Path(candidate.getPath(),
+                      child.getPath().getName()));
+              if (globFilter.accept(child.getPath())) {
+                newCandidates.add(child);
+              }
+            }
+          } else {
+            Path p = new Path(candidate.getPath(), unquotePathComponent(component));
+            FileStatus s = getFileLinkStatus(p);
+            if (s != null) {
+              s.setPath(p);
+              newCandidates.add(s);
             }
           }
         }

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java Tue Aug 20 20:19:07 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Options.Rena
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -632,6 +633,20 @@ public abstract class FileContextMainOpe
         filteredPaths));
   }
   
+  protected Path getHiddenPathForTest() {
+    return null;
+  }
+  
+  @Test
+  public void testGlobStatusFilterWithHiddenPathTrivialFilter()
+      throws Exception {
+    Path hidden = getHiddenPathForTest();
+    Assume.assumeNotNull(hidden);
+    FileStatus[] filteredPaths = fc.util().globStatus(hidden, DEFAULT_FILTER);
+    Assert.assertNotNull(filteredPaths);
+    Assert.assertEquals(1, filteredPaths.length);
+  }
+
   @Test
   public void testWriteReadAndDeleteEmptyFile() throws Exception {
     writeReadAndDelete(0);

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java Tue Aug 20 20:19:07 2013
@@ -517,6 +517,26 @@ public class TestFsShellReturnCode {
       }
       return stat;
     }
+
+    @Override
+    public FileStatus getFileLinkStatus(Path p) throws IOException {
+      String f = makeQualified(p).toString();
+      FileStatus stat = super.getFileLinkStatus(p);
+      
+      stat.getPermission();
+      if (owners.containsKey(f)) {
+        stat.setOwner("STUB-"+owners.get(f));      
+      } else {
+        stat.setOwner("REAL-"+stat.getOwner());
+      }
+      if (groups.containsKey(f)) {
+        stat.setGroup("STUB-"+groups.get(f));      
+      } else {
+        stat.setGroup("REAL-"+stat.getGroup());
+      }
+      return stat;
+    }
+
   }
   
   static class MyFsShell extends FsShell {