Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: common-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1204376 - in
 /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
 src/main/java/org/apache/hadoop/fs/ src/test/java/org/apache/hadoop/fs/
Date: Mon, 21 Nov 2011 07:11:08 -0000
To: common-commits@hadoop.apache.org
From: eli@apache.org
Message-Id: <20111121071108.C19C023889FA@eris.apache.org>

Author: eli
Date: Mon Nov 21 07:11:07 2011
New Revision: 1204376

URL: http://svn.apache.org/viewvc?rev=1204376&view=rev
Log:
HADOOP-7783. Add more symlink tests that cover intermediate links. Contributed by Eli Collins

Modified:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1204376&r1=1204375&r2=1204376&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov 21 07:11:07 2011
@@ -1364,6 +1364,8 @@ Release 0.22.0 - Unreleased
     HADOOP-7457. Remove out-of-date Chinese language documentation.
     (Jakob Homan via eli)
 
+    HADOOP-7783. Add more symlink tests that cover intermediate links. (eli)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1204376&r1=1204375&r2=1204376&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Mon Nov 21 07:11:07 2011
@@ -385,7 +385,7 @@ public abstract class AbstractFileSystem
     checkPath(p);
     String s = p.toUri().getPath();
     if (!isValidName(s)) {
-      throw new InvalidPathException("Path part " + s + " from URI" + p
+      throw new InvalidPathException("Path part " + s + " from URI " + p
           + " is not a valid filename.");
     }
     return s;

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1204376&r1=1204375&r2=1204376&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Mon Nov 21 07:11:07 2011
@@ -1092,29 +1092,28 @@ public final class FileContext {
    * Return a fully qualified version of the given symlink target if it
    * has no scheme and authority. Partially and fully qualified paths 
    * are returned unmodified.
-   * @param linkFS The AbstractFileSystem of link 
-   * @param link   The path of the symlink
-   * @param target The symlink's target
+   * @param pathFS The AbstractFileSystem of the path
+   * @param pathWithLink Path that contains the symlink
+   * @param target The symlink's absolute target
    * @return Fully qualified version of the target.
    */
-  private Path qualifySymlinkTarget(final AbstractFileSystem linkFS, 
-      Path link, Path target) {
-    /* NB: makeQualified uses link's scheme/authority, if specified, 
-     * and the scheme/authority of linkFS, if not. If link does have
-     * a scheme and authority they should match those of linkFS since
-     * resolve updates the path and file system of a path that contains
-     * links each time a link is encountered.
+  private Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
+    Path pathWithLink, Path target) {
+    /* NB: makeQualified uses the target's scheme and authority, if
+     * specified, and the scheme and authority of pathFS, if not. If
+     * the path does have a scheme and authority we assert they match
+     * those of pathFS since resolve updates the file system of a path
+     * that contains links each time a link is encountered.
      */
-    final String linkScheme = link.toUri().getScheme();
-    final String linkAuth   = link.toUri().getAuthority();
-    if (linkScheme != null && linkAuth != null) {
-      assert linkScheme.equals(linkFS.getUri().getScheme());
-      assert linkAuth.equals(linkFS.getUri().getAuthority());
-    }
-    final boolean justPath = (target.toUri().getScheme() == null &&
-                              target.toUri().getAuthority() == null);
-    return justPath ? target.makeQualified(linkFS.getUri(), link.getParent()) 
-                    : target;
+    final String scheme = target.toUri().getScheme();
+    final String auth   = target.toUri().getAuthority();
+    if (scheme != null && auth != null) {
+      assert scheme.equals(pathFS.getUri().getScheme());
+      assert auth.equals(pathFS.getUri().getAuthority());
+    }
+    return (scheme == null && auth == null)
+      ? target.makeQualified(pathFS.getUri(), pathWithLink.getParent())
+      : target;
   }
   
   /**
@@ -1148,16 +1147,19 @@ public final class FileContext {
   }
   
   /**
-   * Returns the un-interpreted target of the given symbolic link.
-   * Transparently resolves all links up to the final path component.
-   * @param f
+   * Returns the target of the given symbolic link as it was specified
+   * when the link was created.  Links in the path leading up to the
+   * final path component are resolved transparently.
+   *
+   * @param f the path to return the target of
    * @return The un-interpreted target of the symbolic link.
    * 
    * @throws AccessControlException If access is denied
    * @throws FileNotFoundException If path <code>f</code> does not exist
    * @throws UnsupportedFileSystemException If file system for <code>f</code> is
    *           not supported
-   * @throws IOException If an I/O error occurred
+   * @throws IOException If the given path does not refer to a symlink
+   *           or an I/O error occurred
    */
   public Path getLinkTarget(final Path f) throws AccessControlException,
       FileNotFoundException, UnsupportedFileSystemException, IOException {
@@ -1277,7 +1279,7 @@ public final class FileContext {
    * getFsStatus, getFileStatus, exists, and listStatus.
    * 
    * Symlink targets are stored as given to createSymlink, assuming the 
-   * underlying file system is capable of storign a fully qualified URI. 
+   * underlying file system is capable of storing a fully qualified URI.
    * Dangling symlinks are permitted. FileContext supports four types of 
    * symlink targets, and resolves them as follows
    * <pre>

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1204376&r1=1204375&r2=1204376&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Mon Nov 21 07:11:07 2011
@@ -68,13 +68,14 @@ public class Path implements Comparable 
     // Add a slash to parent's path so resolution is compatible with URI's
     URI parentUri = parent.uri;
     String parentPath = parentUri.getPath();
-    if (!(parentPath.equals("/") || parentPath.equals("")))
+    if (!(parentPath.equals("/") || parentPath.equals(""))) {
       try {
         parentUri = new URI(parentUri.getScheme(), parentUri.getAuthority(),
                       parentUri.getPath()+"/", null, parentUri.getFragment());
       } catch (URISyntaxException e) {
         throw new IllegalArgumentException(e);
       }
+    }
     URI resolved = parentUri.resolve(child.uri);
     initialize(resolved.getScheme(), resolved.getAuthority(),
                resolved.getPath(), resolved.getFragment());
@@ -213,7 +214,8 @@ public class Path implements Comparable 
    * There is some ambiguity here. An absolute path is a slash
    * relative name without a scheme or an authority.
    * So either this method was incorrectly named or its
-   * implementation is incorrect.
+   * implementation is incorrect. This method returns true
+   * even if there is a scheme and authority.
    */
   public boolean isAbsolute() {
      return isUriPathAbsolute();
@@ -307,19 +309,16 @@ public class Path implements Comparable 
     return depth;
   }
 
-  
   /**
    *  Returns a qualified path object.
    *  
    *  Deprecated - use {@link #makeQualified(URI, Path)}
    */
- 
   @Deprecated
   public Path makeQualified(FileSystem fs) {
     return makeQualified(fs.getUri(), fs.getWorkingDirectory());
   }
   
-  
   /** Returns a qualified path object. */
   @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
   public Path makeQualified(URI defaultUri, Path workingDir ) {

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java?rev=1204376&r1=1204375&r2=1204376&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java Mon Nov 21 07:11:07 2011
@@ -28,8 +28,9 @@ import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import static org.apache.hadoop.fs.FileContextTestHelper.*;
-import static org.junit.Assert.*;
 
+import static org.junit.Assert.*;
+import static org.junit.Assume.assumeTrue;
 import org.junit.Test;
 import org.junit.Before;
 import org.junit.After;
@@ -238,6 +239,31 @@ public abstract class FileContextSymlink
     assertFalse(isDir(fc, linkToFile));
     assertEquals(file.toUri().getPath(), 
                  fc.getLinkTarget(linkToFile).toString());
+    // The local file system does not fully resolve the link
+    // when obtaining the file status
+    if (!"file".equals(getScheme())) {
+      assertEquals(fc.getFileStatus(file), fc.getFileStatus(linkToFile));
+      assertEquals(fc.makeQualified(file),
+                   fc.getFileStatus(linkToFile).getPath());
+      assertEquals(fc.makeQualified(linkToFile),
+                   fc.getFileLinkStatus(linkToFile).getPath());
+    }
+  }
+
+  @Test
+  /** Stat a relative link to a file */
+  public void testStatRelLinkToFile() throws IOException {
+    assumeTrue(!"file".equals(getScheme()));
+    Path baseDir    = new Path(testBaseDir1());
+    Path file       = new Path(testBaseDir1(), "file");
+    Path linkToFile = new Path(testBaseDir1(), "linkToFile");
+    createAndWriteFile(file);
+    fc.createSymlink(new Path("file"), linkToFile, false);
+    assertEquals(fc.getFileStatus(file), fc.getFileStatus(linkToFile));
+    assertEquals(fc.makeQualified(file),
+                 fc.getFileStatus(linkToFile).getPath());
+    assertEquals(fc.makeQualified(linkToFile),
+                 fc.getFileLinkStatus(linkToFile).getPath());
   }
 
   @Test
@@ -474,18 +500,15 @@ public abstract class FileContextSymlink
    * creating using a partially qualified path is file system specific.
    */
   public void testCreateLinkUsingPartQualPath1() throws IOException {
+    // Partially qualified paths are covered for local file systems
+    // in the previous test.
+    assumeTrue(!"file".equals(getScheme()));
     Path schemeAuth   = new Path(testURI().toString());
     Path fileWoHost   = new Path(getScheme()+"://"+testBaseDir1()+"/file");
     Path link         = new Path(testBaseDir1()+"/linkToFile");
     Path linkQual     = new Path(schemeAuth, testBaseDir1()+"/linkToFile");
-    
-    // Partially qualified paths are covered for local file systems
-    // in the previous test.
-    if ("file".equals(getScheme())) {
-      return;
-    }
     FileContext localFc = FileContext.getLocalFSFileContext();
-    
+
     fc.createSymlink(fileWoHost, link, false);
     // Partially qualified path is stored
     assertEquals(fileWoHost, fc.getLinkTarget(linkQual));    
@@ -748,7 +771,7 @@ public abstract class FileContextSymlink
   }
 
   @Test
-  /** Test create symlink to ../foo */
+  /** Test create symlink to ../file */
   public void testCreateLinkToDotDotPrefix() throws IOException {
     Path file = new Path(testBaseDir1(), "file");
     Path dir  = new Path(testBaseDir1(), "test");
@@ -1205,24 +1228,30 @@ public abstract class FileContextSymlink
   }
   
   @Test
-  /** Operate on a file using a path with an intermediate symlink */  
-  public void testAccessFileViaSymlink() throws IOException {
+  /**
+   * Create, write, read, append, rename, get the block locations,
+   * checksums, and delete a file using a path with a symlink as an
+   * intermediate path component where the link target was specified
+   * using an absolute path. Rename is covered in more depth below.
+   */
+  public void testAccessFileViaInterSymlinkAbsTarget() throws IOException {
     Path baseDir        = new Path(testBaseDir1());
+    Path file           = new Path(testBaseDir1(), "file");
     Path fileNew        = new Path(baseDir, "fileNew");
     Path linkToDir      = new Path(testBaseDir2(), "linkToDir");
     Path fileViaLink    = new Path(linkToDir, "file");
     Path fileNewViaLink = new Path(linkToDir, "fileNew");
     fc.createSymlink(baseDir, linkToDir, false);
-    // Create, write, read, append, rename, get block locations and 
-    // checksums, and delete a file using a path that contains a 
-    // symlink as an intermediate path component. Rename is covered 
-    // in more depth below.
     createAndWriteFile(fileViaLink);
     assertTrue(exists(fc, fileViaLink));
     assertTrue(isFile(fc, fileViaLink));
     assertFalse(isDir(fc, fileViaLink));
     assertFalse(fc.getFileLinkStatus(fileViaLink).isSymlink());
     assertFalse(isDir(fc, fileViaLink));
+    assertEquals(fc.getFileStatus(file),
+                 fc.getFileLinkStatus(file));
+    assertEquals(fc.getFileStatus(fileViaLink),
+                 fc.getFileLinkStatus(fileViaLink));
     readFile(fileViaLink);
     appendToFile(fileViaLink);
     fc.rename(fileViaLink, fileNewViaLink);
@@ -1238,6 +1267,58 @@ public abstract class FileContextSymlink
   }
 
   @Test
+  /**
+   * Operate on a file using a path with an intermediate symlink where
+   * the link target was specified as a fully qualified path.
+   */
+  public void testAccessFileViaInterSymlinkQualTarget() throws IOException {
+    Path baseDir        = new Path(testBaseDir1());
+    Path file           = new Path(testBaseDir1(), "file");
+    Path fileNew        = new Path(baseDir, "fileNew");
+    Path linkToDir      = new Path(testBaseDir2(), "linkToDir");
+    Path fileViaLink    = new Path(linkToDir, "file");
+    Path fileNewViaLink = new Path(linkToDir, "fileNew");
+    fc.createSymlink(fc.makeQualified(baseDir), linkToDir, false);
+    createAndWriteFile(fileViaLink);
+    assertEquals(fc.getFileStatus(file),
+                 fc.getFileLinkStatus(file));
+    assertEquals(fc.getFileStatus(fileViaLink),
+                 fc.getFileLinkStatus(fileViaLink));
+    readFile(fileViaLink);
+  }
+
+  @Test
+  /**
+   * Operate on a file using a path with an intermediate symlink where
+   * the link target was specified as a relative path.
+   */
+  public void testAccessFileViaInterSymlinkRelTarget() throws IOException {
+    assumeTrue(!"file".equals(getScheme()));
+    Path baseDir     = new Path(testBaseDir1());
+    Path dir         = new Path(testBaseDir1(), "dir");
+    Path file        = new Path(dir, "file");
+    Path linkToDir   = new Path(testBaseDir1(), "linkToDir");
+    Path fileViaLink = new Path(linkToDir, "file");
+
+    fc.mkdir(dir, FileContext.DEFAULT_PERM, false);
+    fc.createSymlink(new Path("dir"), linkToDir, false);
+    createAndWriteFile(fileViaLink);
+    // Note that getFileStatus returns fully qualified paths even
+    // when called on an absolute path.
+    assertEquals(fc.makeQualified(file),
+                 fc.getFileStatus(file).getPath());
+    // In each case getFileLinkStatus returns the same FileStatus
+    // as getFileStatus since we're not calling it on a link and
+    // FileStatus objects are compared by Path.
+    assertEquals(fc.getFileStatus(file),
+                 fc.getFileLinkStatus(file));
+    assertEquals(fc.getFileStatus(fileViaLink),
+                 fc.getFileLinkStatus(fileViaLink));
+    assertEquals(fc.getFileStatus(fileViaLink),
+                 fc.getFileLinkStatus(file));
+  }
+
+  @Test
   /** Test create, list, and delete a directory through a symlink */
   public void testAccessDirViaSymlink() throws IOException {
     Path baseDir    = new Path(testBaseDir1());
@@ -1272,4 +1353,4 @@ public abstract class FileContextSymlink
       assertEquals(2, fc.getFileStatus(file).getModificationTime());
     }
   }
-}
\ No newline at end of file
+}

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java?rev=1204376&r1=1204375&r2=1204376&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java Mon Nov 21 07:11:07 2011
@@ -95,6 +95,7 @@ public class TestPath extends TestCase {
     assertEquals(new Path("/foo"), new Path("/foo/bar").getParent());
     assertEquals(new Path("foo"), new Path("foo/bar").getParent());
     assertEquals(new Path("/"), new Path("/foo").getParent());
+    assertEquals(null, new Path("/").getParent());
     if (Path.WINDOWS) {
       assertEquals(new Path("c:/"), new Path("c:/foo").getParent());
     }
@@ -159,6 +160,13 @@ public class TestPath extends TestCase {
     assertEquals(new Path("foo/bar/baz","../../..").toString(), "");
     assertEquals(new Path("foo/bar/baz","../../../../..").toString(), "../..");
   }
+
+  /** Test Path objects created from other Path objects */
+  public void testChildParentResolution() throws URISyntaxException, IOException {
+    Path parent = new Path("foo1://bar1/baz1");
+    Path child  = new Path("foo2://bar2/baz2");
+    assertEquals(child, new Path(parent, child));
+  }
   
   public void testScheme() throws java.io.IOException {
     assertEquals("foo:/bar", new Path("foo:/","/bar").toString());