hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1461898 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/test/java/org/apache/hadoop/fs/
Date Wed, 27 Mar 2013 23:43:45 GMT
Author: eli
Date: Wed Mar 27 23:43:45 2013
New Revision: 1461898

URL: http://svn.apache.org/r1461898
Log:
HADOOP-9357. Fallback to default authority if not specified in FileContext. Contributed by
Andrew Wang

Modified:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1461898&r1=1461897&r2=1461898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Mar 27 23:43:45
2013
@@ -601,6 +601,9 @@ Release 2.0.5-beta - UNRELEASED
     HADOOP-9125. LdapGroupsMapping threw CommunicationException after some
     idle time. (Kai Zheng via atm)
 
+    HADOOP-9357. Fallback to default authority if not specified in FileContext.
+    (Andrew Wang via eli)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1461898&r1=1461897&r2=1461898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
Wed Mar 27 23:43:45 2013
@@ -244,17 +244,33 @@ public final class FileContext {
   }
  
   /* 
-   * Remove relative part - return "absolute":
-   * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
-   * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
+   * Resolve a relative path passed from the user.
+   * 
+   * Relative paths are resolved against the current working directory
+   * (e.g. "foo/bar" becomes "/<workingDir>/foo/bar").
+   * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths
    * ("/foo/bar") are returned unchanged.
    * 
+   * Additionally, we fix malformed URIs that specify a scheme but not an 
+   * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme
+   * if it matches the default FS, and let the default FS add in the default
+   * scheme and authority later (see {@link #AbstractFileSystem#checkPath}).
+   * 
    * Applications that use FileContext should use #makeQualified() since
-   * they really want a fully qualified URI.
+   * they really want a fully-qualified URI.
    * Hence this method is not called makeAbsolute() and 
    * has been deliberately declared private.
    */
   private Path fixRelativePart(Path p) {
+    // Per RFC 2396 5.2, drop schema if there is a scheme but no authority.
+    if (p.hasSchemeAndNoAuthority()) {
+      String scheme = p.toUri().getScheme();
+      if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) {
+        p = new Path(p.toUri().getSchemeSpecificPart());
+      }
+    }
+    // Absolute paths are unchanged. Relative paths are resolved against the
+    // current working directory.
     if (p.isUriPathAbsolute()) {
       return p;
     } else {

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1461898&r1=1461897&r2=1461898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Wed Mar 27 23:43:45 2013
@@ -256,6 +256,10 @@ public class Path implements Comparable 
     return  (isUriPathAbsolute() && 
         uri.getScheme() == null && uri.getAuthority() == null);
   }
+
+  public boolean hasSchemeAndNoAuthority() {
+    return uri.getScheme() != null && uri.getAuthority() == null;
+  }
   
   /**
    *  True if the path component (i.e. directory) of this URI is absolute.

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java?rev=1461898&r1=1461897&r2=1461898&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
Wed Mar 27 23:43:45 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.EnumSet;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -1164,6 +1166,40 @@ public abstract class FileContextMainOpe
       Assert.assertEquals(fc.getFileStatus(file), fc.getFileLinkStatus(file));
     }
   }
+
+  /**
+   * Test that URIs with a scheme, no authority, and absolute path component
+   * resolve with the authority of the default FS.
+   */
+  @Test(timeout=30000)
+  public void testAbsolutePathSchemeNoAuthority() throws IOException,
+      URISyntaxException {
+    Path file = getTestRootPath(fc, "test/file");
+    createFile(file);
+    URI uri = file.toUri();
+    URI noAuthorityUri = new URI(uri.getScheme(), null, uri.getPath(),
+        uri.getQuery(), uri.getFragment());
+    Path noAuthority = new Path(noAuthorityUri);
+    Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
+  }
+
+  /**
+   * Test that URIs with a scheme, no authority, and relative path component
+   * resolve with the authority of the default FS.
+   */
+  @Test(timeout=30000)
+  public void testRelativePathSchemeNoAuthority() throws IOException,
+      URISyntaxException {
+    Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test"));
+    fc.setWorkingDirectory(workDir);
+    Path file = new Path(workDir, "file");
+    createFile(file);
+    URI uri = file.toUri();
+    URI noAuthorityUri = new URI(uri.getScheme() + ":file");
+    System.out.println(noAuthorityUri);
+    Path noAuthority = new Path(noAuthorityUri);
+    Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
+  }
   
   protected void createFile(Path path) throws IOException {
     FSDataOutputStream out = fc.create(path, EnumSet.of(CREATE),



Mime
View raw message