hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1498720 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/fs/local/
Date Mon, 01 Jul 2013 22:09:52 GMT
Author: cmccabe
Date: Mon Jul  1 22:09:52 2013
New Revision: 1498720

URL: http://svn.apache.org/r1498720
Log:
HADOOP-9414.  Refactor out FSLinkResolver and relevant helper methods.

Added:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
Modified:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1498720&r1=1498719&r2=1498720&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Jul  1 22:09:52
2013
@@ -306,6 +306,9 @@ Release 2.2.0 - UNRELEASED
     HADOOP-9673.  NetworkTopology: when a node can't be added, print out its
     location for diagnostic purposes.  (Colin Patrick McCabe)
 
+    HADOOP-9414.  Refactor out FSLinkResolver and relevant helper methods.
+    (Andrew Wang via Colin Patrick McCabe)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1498720&r1=1498719&r2=1498720&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
Mon Jul  1 22:09:52 2013
@@ -729,6 +729,7 @@ public abstract class AbstractFileSystem
   
   /**
    * Returns true if the file system supports symlinks, false otherwise.
+   * @return true if filesystem supports symlinks
    */
   public boolean supportsSymlinks() {
     return false;
@@ -744,8 +745,9 @@ public abstract class AbstractFileSystem
   }
 
   /**
-   * The specification of this method matches that of  
-   * {@link FileContext#getLinkTarget(Path)};
+   * Partially resolves the path. This is used during symlink resolution in
+   * {@link FSLinkResolver}, and differs from the similarly named method
+   * {@link FileContext#getLinkTarget(Path)}.
    */
   public Path getLinkTarget(final Path f) throws IOException {
     /* We should never get here. Any file system that threw an

Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java?rev=1498720&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
(added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
Mon Jul  1 22:09:52 2013
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * Class used to perform an operation on and resolve symlinks in a
+ * path. The operation may potentially span multiple file systems.
+ */
+public abstract class FSLinkResolver<T> {
+
+  private static final int MAX_PATH_LINKS = 32;
+
+  /**
+   * See {@link #qualifySymlinkTarget(URI, Path, Path)}
+   */
+  public static Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
+    Path pathWithLink, Path target) {
+    return qualifySymlinkTarget(pathFS.getUri(), pathWithLink, target);
+  }
+
+  /**
+   * See {@link #qualifySymlinkTarget(URI, Path, Path)}
+   */
+  public static Path qualifySymlinkTarget(final FileSystem pathFS,
+      Path pathWithLink, Path target) {
+    return qualifySymlinkTarget(pathFS.getUri(), pathWithLink, target);
+  }
+  
+  /**
+   * Return a fully-qualified version of the given symlink target if it
+   * has no scheme and authority. Partially and fully-qualified paths
+   * are returned unmodified.
+   * @param pathURI URI of the filesystem of pathWithLink
+   * @param pathWithLink Path that contains the symlink
+   * @param target The symlink's absolute target
+   * @return Fully qualified version of the target.
+   */
+  private static Path qualifySymlinkTarget(final URI pathURI,
+      Path pathWithLink, Path target) {
+    // NB: makeQualified uses the target's scheme and authority, if
+    // specified, and the scheme and authority of pathURI, if not.
+    final URI targetUri = target.toUri();
+    final String scheme = targetUri.getScheme();
+    final String auth = targetUri.getAuthority();
+    return (scheme == null && auth == null) ? target.makeQualified(pathURI,
+        pathWithLink.getParent()) : target;
+  }
+
+  // FileContext / AbstractFileSystem resolution methods
+
+  /**
+   * Generic helper function overridden on instantiation to perform a
+   * specific operation on the given file system using the given path
+   * which may result in an UnresolvedLinkException.
+   * @param fs AbstractFileSystem to perform the operation on.
+   * @param p Path given the file system.
+   * @return Generic type determined by the specific implementation.
+   * @throws UnresolvedLinkException If symbolic link <code>path</code> could
+   *           not be resolved
+   * @throws IOException an I/O error occurred
+   */
+  public T next(final AbstractFileSystem fs, final Path p)
+      throws IOException, UnresolvedLinkException {
+    throw new AssertionError("Should not be called without first overriding!");
+  }
+
+  /**
+   * Performs the operation specified by the next function, calling it
+   * repeatedly until all symlinks in the given path are resolved.
+   * @param fc FileContext used to access file systems.
+   * @param path The path to resolve symlinks on.
+   * @return Generic type determined by the implementation of next.
+   * @throws IOException
+   */
+  public T resolve(final FileContext fc, final Path path) throws IOException {
+    int count = 0;
+    T in = null;
+    Path p = path;
+    // NB: More than one AbstractFileSystem can match a scheme, eg 
+    // "file" resolves to LocalFs but could have come by RawLocalFs.
+    AbstractFileSystem fs = fc.getFSofPath(p);
+
+    // Loop until all symlinks are resolved or the limit is reached
+    for (boolean isLink = true; isLink;) {
+      try {
+        in = next(fs, p);
+        isLink = false;
+      } catch (UnresolvedLinkException e) {
+        if (count++ > MAX_PATH_LINKS) {
+          throw new IOException("Possible cyclic loop while " +
+                                "following symbolic link " + path);
+        }
+        // Resolve the first unresolved path component
+        p = FSLinkResolver.qualifySymlinkTarget(fs, p, fs.getLinkTarget(p));
+        fs = fc.getFSofPath(p);
+      }
+    }
+    return in;
+  }
+}

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1498720&r1=1498719&r2=1498720&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
Mon Jul  1 22:09:52 2013
@@ -282,17 +282,6 @@ public final class FileContext {
       DELETE_ON_EXIT.clear();
     }
   }
-  
-  /**
-   * Pathnames with scheme and relative path are illegal.
-   * @param path to be checked
-   */
-  private static void checkNotSchemeWithRelative(final Path path) {
-    if (path.toUri().isAbsolute() && !path.isUriPathAbsolute()) {
-      throw new HadoopIllegalArgumentException(
-          "Unsupported name: has scheme but relative path-part");
-    }
-  }
 
   /**
    * Get the file system of supplied path.
@@ -305,13 +294,10 @@ public final class FileContext {
    * @throws IOExcepton If the file system for <code>absOrFqPath</code> could
    *         not be instantiated.
    */
-  private AbstractFileSystem getFSofPath(final Path absOrFqPath)
+  protected AbstractFileSystem getFSofPath(final Path absOrFqPath)
       throws UnsupportedFileSystemException, IOException {
-    checkNotSchemeWithRelative(absOrFqPath);
-    if (!absOrFqPath.isAbsolute() && absOrFqPath.toUri().getScheme() == null) {
-      throw new HadoopIllegalArgumentException(
-          "FileContext Bug: path is relative");
-    }
+    absOrFqPath.checkNotSchemeWithRelative();
+    absOrFqPath.checkNotRelative();
 
     try { 
       // Is it the default FS for this FileContext?
@@ -513,7 +499,7 @@ public final class FileContext {
    *           </ul>
    */
   public void setWorkingDirectory(final Path newWDir) throws IOException {
-    checkNotSchemeWithRelative(newWDir);
+    newWDir.checkNotSchemeWithRelative();
     /* wd is stored as a fully qualified path. We check if the given 
      * path is not relative first since resolve requires and returns 
      * an absolute path.
@@ -1119,26 +1105,6 @@ public final class FileContext {
   }
 
   /**
-   * Return a fully qualified version of the given symlink target if it
-   * has no scheme and authority. Partially and fully qualified paths 
-   * are returned unmodified.
-   * @param pathFS The AbstractFileSystem of the path
-   * @param pathWithLink Path that contains the symlink
-   * @param target The symlink's absolute target
-   * @return Fully qualified version of the target.
-   */
-  private static Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
-    Path pathWithLink, Path target) {
-    // NB: makeQualified uses the target's scheme and authority, if
-    // specified, and the scheme and authority of pathFS, if not.
-    final String scheme = target.toUri().getScheme();
-    final String auth   = target.toUri().getAuthority();
-    return (scheme == null && auth == null)
-      ? target.makeQualified(pathFS.getUri(), pathWithLink.getParent())
-      : target;
-  }
-  
-  /**
    * Return a file status object that represents the path. If the path 
    * refers to a symlink then the FileStatus of the symlink is returned.
    * The behavior is equivalent to #getFileStatus() if the underlying
@@ -2156,9 +2122,9 @@ public final class FileContext {
         boolean overwrite) throws AccessControlException,
         FileAlreadyExistsException, FileNotFoundException,
         ParentNotDirectoryException, UnsupportedFileSystemException, 
-	IOException {
-      checkNotSchemeWithRelative(src);
-      checkNotSchemeWithRelative(dst);
+        IOException {
+      src.checkNotSchemeWithRelative();
+      dst.checkNotSchemeWithRelative();
       Path qSrc = makeQualified(src);
       Path qDst = makeQualified(dst);
       checkDest(qSrc.getName(), qDst, overwrite);
@@ -2324,64 +2290,7 @@ public final class FileContext {
     }.resolve(this, absF);
     return result;
   }
-  
-  /**
-   * Class used to perform an operation on and resolve symlinks in a
-   * path. The operation may potentially span multiple file systems.  
-   */
-  protected static abstract class FSLinkResolver<T> {
-    // The maximum number of symbolic link components in a path
-    private static final int MAX_PATH_LINKS = 32;
-
-    /**
-     * Generic helper function overridden on instantiation to perform a 
-     * specific operation on the given file system using the given path
-     * which may result in an UnresolvedLinkException. 
-     * @param fs AbstractFileSystem to perform the operation on.
-     * @param p Path given the file system.
-     * @return Generic type determined by the specific implementation.
-     * @throws UnresolvedLinkException If symbolic link <code>path</code> could

-     *           not be resolved
-     * @throws IOException an I/O error occured
-     */
-    public abstract T next(final AbstractFileSystem fs, final Path p) 
-      throws IOException, UnresolvedLinkException;  
-        
-    /**
-     * Performs the operation specified by the next function, calling it
-     * repeatedly until all symlinks in the given path are resolved.
-     * @param fc FileContext used to access file systems.
-     * @param p The path to resolve symlinks in.
-     * @return Generic type determined by the implementation of next.
-     * @throws IOException
-     */
-    public T resolve(final FileContext fc, Path p) throws IOException {
-      int count = 0;
-      T in = null;
-      Path first = p;
-      // NB: More than one AbstractFileSystem can match a scheme, eg 
-      // "file" resolves to LocalFs but could have come by RawLocalFs.
-      AbstractFileSystem fs = fc.getFSofPath(p);      
 
-      // Loop until all symlinks are resolved or the limit is reached
-      for (boolean isLink = true; isLink;) {
-        try {
-          in = next(fs, p);
-          isLink = false;
-        } catch (UnresolvedLinkException e) {
-          if (count++ > MAX_PATH_LINKS) {
-            throw new IOException("Possible cyclic loop while " +
-                                  "following symbolic link " + first);
-          }
-          // Resolve the first unresolved path component
-          p = qualifySymlinkTarget(fs, p, fs.getLinkTarget(p));
-          fs = fc.getFSofPath(p);
-        }
-      }
-      return in;
-    }
-  }
-  
   /**
    * Get the statistics for a particular file system
    * 

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1498720&r1=1498719&r2=1498720&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
Mon Jul  1 22:09:52 2013
@@ -25,6 +25,7 @@ import java.util.regex.Pattern;
 
 import org.apache.avro.reflect.Stringable;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -57,6 +58,32 @@ public class Path implements Comparable 
 
   private URI uri;                                // a hierarchical uri
 
+  /**
+   * Pathnames with scheme and relative path are illegal.
+   * @param path to be checked
+   */
+  void checkNotSchemeWithRelative() {
+    if (toUri().isAbsolute() && !isUriPathAbsolute()) {
+      throw new HadoopIllegalArgumentException(
+          "Unsupported name: has scheme but relative path-part");
+    }
+  }
+
+  void checkNotRelative() {
+    if (!isAbsolute() && toUri().getScheme() == null) {
+      throw new HadoopIllegalArgumentException("Path is relative");
+    }
+  }
+
+  public static Path getPathWithoutSchemeAndAuthority(Path path) {
+    // This code depends on Path.toString() to remove the leading slash before
+    // the drive specification on Windows.
+    Path newPath = path.isUriPathAbsolute() ?
+      new Path(null, null, path.toUri().getPath()) :
+      path;
+    return newPath;
+  }
+
   /** Resolve a child path against a parent path. */
   public Path(String parent, String child) {
     this(new Path(parent), new Path(child));

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1498720&r1=1498719&r2=1498720&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
Mon Jul  1 22:09:52 2013
@@ -17,20 +17,20 @@
  */
 package org.apache.hadoop.fs.local;
 
-import java.io.IOException;
-import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FsConstants;
 import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.util.Shell;
@@ -91,8 +91,8 @@ public class RawLocalFs extends Delegate
     // NB: Use createSymbolicLink in java.nio.file.Path once available
     try {
       Shell.execCommand(Shell.getSymlinkCommand(
-        getPathWithoutSchemeAndAuthority(target).getPath(),
-        getPathWithoutSchemeAndAuthority(link).getPath()));
+        Path.getPathWithoutSchemeAndAuthority(target).toString(),
+        Path.getPathWithoutSchemeAndAuthority(link).toString()));
     } catch (IOException x) {
       throw new IOException("Unable to create symlink: "+x.getMessage());
     }
@@ -175,13 +175,4 @@ public class RawLocalFs extends Delegate
      */
     throw new AssertionError();
   }
-
-  private static File getPathWithoutSchemeAndAuthority(Path path) {
-    Path newPath = path.isUriPathAbsolute() ?
-      new Path(null, null, path.toUri().getPath()) :
-      path;
-
-    // Path.toString() removes leading slash before drive spec on Windows.
-    return new File(newPath.toString());
-  }
 }



Mime
View raw message