hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1300292 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/FileSystem.java src/main/java/org/apache/hadoop/fs/Path.java src/test/java/org/apache/hadoop/fs/TestPath.java
Date Tue, 13 Mar 2012 19:10:41 GMT
Author: suresh
Date: Tue Mar 13 19:10:41 2012
New Revision: 1300292

URL: http://svn.apache.org/viewvc?rev=1300292&view=rev
Log:
HADOOP-8164. Merging change 1300290 from trunk to 0.23

Modified:
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1300292&r1=1300291&r2=1300292&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Tue
Mar 13 19:10:41 2012
@@ -265,6 +265,9 @@ Release 0.23.2 - UNRELEASED 
     HADOOP-8140. dfs -getmerge should process its argments better (Daryn Sharp
     via bobby)
 
+    HADOOP-8164. Back slash as path separator is handled for Windows only.
+    (Daryn Sharp via suresh)
+
 Release 0.23.1 - 2012-02-17 
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1300292&r1=1300291&r2=1300292&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
Tue Mar 13 19:10:41 2012
@@ -1456,11 +1456,12 @@ public abstract class FileSystem extends
         results = listStatus(parentPaths, fp);
         hasGlob[0] = true;
       } else { // last component does not have a pattern
+        // remove the quoting of metachars in a non-regexp expansion
+        String name = unquotePathComponent(components[components.length - 1]);
         // get all the path names
         ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length);
         for (int i = 0; i < parentPaths.length; i++) {
-          parentPaths[i] = new Path(parentPaths[i],
-            components[components.length - 1]);
+          parentPaths[i] = new Path(parentPaths[i], name);
           if (fp.accept(parentPaths[i])) {
             filteredPaths.add(parentPaths[i]);
           }
@@ -1503,15 +1504,29 @@ public abstract class FileSystem extends
     if (fp.hasPattern()) {
       parents = FileUtil.stat2Paths(listStatus(parents, fp));
       hasGlob[0] = true;
-    } else {
+    } else { // the component does not have a pattern
+      // remove the quoting of metachars in a non-regexp expansion
+      String name = unquotePathComponent(filePattern[level]);
       for (int i = 0; i < parents.length; i++) {
-        parents[i] = new Path(parents[i], filePattern[level]);
+        parents[i] = new Path(parents[i], name);
       }
     }
     return globPathsLevel(parents, filePattern, level + 1, hasGlob);
   }
 
   /**
+   * The glob filter builds a regexp per path component.  If the component
+   * does not contain a shell metachar, then it falls back to appending the
+   * raw string to the list of built up paths.  This raw path needs to have
+   * the quoting removed.  Ie. convert all occurances of "\X" to "X"
+   * @param name of the path component
+   * @return the unquoted path component
+   */
+  private String unquotePathComponent(String name) {
+    return name.replaceAll("\\\\(.)", "$1");
+  }
+  
+  /**
    * List the statuses of the files/directories in the given path if the path is
    * a directory. 
    * Return the file's status and block locations If the path is a file.

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1300292&r1=1300291&r2=1300292&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Tue Mar 13 19:10:41 2012
@@ -161,7 +161,9 @@ public class Path implements Comparable 
   private String normalizePath(String path) {
     // remove double slashes & backslashes
     path = StringUtils.replace(path, "//", "/");
-    path = StringUtils.replace(path, "\\", "/");
+    if (Path.WINDOWS) {
+      path = StringUtils.replace(path, "\\", "/");
+    }
     
     // trim trailing slash from non-root path (ignoring windows drive)
     int minLength = hasWindowsDrive(path, true) ? 4 : 1;

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java?rev=1300292&r1=1300291&r2=1300292&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
Tue Mar 13 19:10:41 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.AvroTestUtil;
@@ -264,4 +265,61 @@ public class TestPath extends TestCase {
        "{\"type\":\"string\",\"java-class\":\"org.apache.hadoop.fs.Path\"}");
   }
 
+  public void testGlobEscapeStatus() throws Exception {
+    FileSystem lfs = FileSystem.getLocal(new Configuration());
+    Path testRoot = lfs.makeQualified(new Path(
+        System.getProperty("test.build.data","test/build/data"),
+        "testPathGlob"));
+    lfs.delete(testRoot, true);
+    lfs.mkdirs(testRoot);
+    assertTrue(lfs.isDirectory(testRoot));
+    lfs.setWorkingDirectory(testRoot);
+    
+    // create a couple dirs with file in them
+    Path paths[] = new Path[]{
+        new Path(testRoot, "*/f"),
+        new Path(testRoot, "d1/f"),
+        new Path(testRoot, "d2/f")
+    };
+    Arrays.sort(paths);
+    for (Path p : paths) {
+      lfs.create(p).close();
+      assertTrue(lfs.exists(p));
+    }
+
+    // try the non-globbed listStatus
+    FileStatus stats[] = lfs.listStatus(new Path(testRoot, "*"));
+    assertEquals(1, stats.length);
+    assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());
+
+    // ensure globStatus with "*" finds all dir contents
+    stats = lfs.globStatus(new Path(testRoot, "*"));
+    Arrays.sort(stats);
+    assertEquals(paths.length, stats.length);
+    for (int i=0; i < paths.length; i++) {
+      assertEquals(paths[i].getParent(), stats[i].getPath());
+    }
+
+    // ensure that globStatus with an escaped "\*" only finds "*"
+    stats = lfs.globStatus(new Path(testRoot, "\\*"));
+    assertEquals(1, stats.length);
+    assertEquals(new Path(testRoot, "*"), stats[0].getPath());
+
+    // try to glob the inner file for all dirs
+    stats = lfs.globStatus(new Path(testRoot, "*/f"));
+    assertEquals(paths.length, stats.length);
+    for (int i=0; i < paths.length; i++) {
+      assertEquals(paths[i], stats[i].getPath());
+    }
+
+    // try to get the inner file for only the "*" dir
+    stats = lfs.globStatus(new Path(testRoot, "\\*/f"));
+    assertEquals(1, stats.length);
+    assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());
+
+    // try to glob all the contents of the "*" dir
+    stats = lfs.globStatus(new Path(testRoot, "\\*/*"));
+    assertEquals(1, stats.length);
+    assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());
+  }
 }



Mime
View raw message