hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1508248 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/fs/local/ src/main/java/org/apache/hadoop/util/ src/test/java/org/apache/hadoop/fs/
Date Mon, 29 Jul 2013 23:57:00 GMT
Author: cmccabe
Date: Mon Jul 29 23:56:59 2013
New Revision: 1508248

URL: http://svn.apache.org/r1508248
Log:
revert HADOOP-9652

Removed:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
Modified:
    hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
    hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Jul 29 23:56:59
2013
@@ -310,9 +310,6 @@ Release 2.3.0 - UNRELEASED
     HADOOP-9582. Non-existent file to "hadoop fs -conf" doesn't throw error
     (Ashwin Shankar via jlowe)
 
-    HADOOP-9652.  RawLocalFs#getFileLinkStatus does not fill in the link owner
-    and mode.  (Andrew Wang via Colin Patrick McCabe)
-
 Release 2.1.1-beta - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
Mon Jul 29 23:56:59 2013
@@ -113,14 +113,7 @@ public abstract class DelegateToFileSyst
 
   @Override
   public FileStatus getFileLinkStatus(final Path f) throws IOException {
-    FileStatus status = fsImpl.getFileLinkStatus(f);
-    // FileSystem#getFileLinkStatus qualifies the link target
-    // AbstractFileSystem needs to return it plain since it's qualified
-    // in FileContext, so re-get and set the plain target
-    if (status.isSymlink()) {
-      status.setSymlink(fsImpl.getLinkTarget(f));
-    }
-    return status;
+    return getFileStatus(f);
   }
 
   @Override
@@ -206,18 +199,22 @@ public abstract class DelegateToFileSyst
 
   @Override
   public boolean supportsSymlinks() {
-    return fsImpl.supportsSymlinks();
+    return false;
   }  
   
   @Override
   public void createSymlink(Path target, Path link, boolean createParent) 
       throws IOException { 
-    fsImpl.createSymlink(target, link, createParent);
+    throw new IOException("File system does not support symlinks");
   } 
   
   @Override
   public Path getLinkTarget(final Path f) throws IOException {
-    return fsImpl.getLinkTarget(f);
+    /* We should never get here. Any file system that threw an 
+     * UnresolvedLinkException, causing this function to be called,
+     * should override getLinkTarget. 
+     */
+    throw new AssertionError();
   }
 
   @Override //AbstractFileSystem

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
Mon Jul 29 23:56:59 2013
@@ -26,7 +26,6 @@ import java.io.InputStreamReader;
 import java.util.Arrays;
 
 import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.Shell.OSType;
 
 /**
  * Class for creating hardlinks.
@@ -42,7 +41,15 @@ import org.apache.hadoop.util.Shell.OSTy
  */
 public class HardLink { 
 
-  public static OSType osType = Shell.osType;
+  public enum OSType {
+    OS_TYPE_UNIX,
+    OS_TYPE_WIN,
+    OS_TYPE_SOLARIS,
+    OS_TYPE_MAC,
+    OS_TYPE_FREEBSD
+  }
+  
+  public static OSType osType;
   private static HardLinkCommandGetter getHardLinkCommand;
   
   public final LinkStats linkStats; //not static
@@ -50,11 +57,12 @@ public class HardLink { 
   //initialize the command "getters" statically, so can use their 
   //methods without instantiating the HardLink object
   static { 
+    osType = getOSType();
     if (osType == OSType.OS_TYPE_WIN) {
       // Windows
       getHardLinkCommand = new HardLinkCGWin();
     } else {
-      // Unix or Linux
+      // Unix
       getHardLinkCommand = new HardLinkCGUnix();
       //override getLinkCountCommand for the particular Unix variant
       //Linux is already set as the default - {"stat","-c%h", null}
@@ -71,7 +79,27 @@ public class HardLink { 
   public HardLink() {
     linkStats = new LinkStats();
   }
-
+  
+  static private OSType getOSType() {
+    String osName = System.getProperty("os.name");
+    if (Shell.WINDOWS) {
+      return OSType.OS_TYPE_WIN;
+    }
+    else if (osName.contains("SunOS") 
+            || osName.contains("Solaris")) {
+       return OSType.OS_TYPE_SOLARIS;
+    }
+    else if (osName.contains("Mac")) {
+       return OSType.OS_TYPE_MAC;
+    }
+    else if (osName.contains("FreeBSD")) {
+       return OSType.OS_TYPE_FREEBSD;
+    }
+    else {
+      return OSType.OS_TYPE_UNIX;
+    }
+  }
+  
   /**
    * This abstract class bridges the OS-dependent implementations of the 
    * needed functionality for creating hardlinks and querying link counts.

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Mon Jul 29 23:56:59 2013
@@ -385,9 +385,6 @@ public class RawLocalFileSystem extends 
       throw new FileNotFoundException("File " + f + " does not exist");
     }
     if (localf.isFile()) {
-      if (!Shell.WINDOWS) {
-        return new FileStatus[] { getFileStatus(f) };
-      }
       return new FileStatus[] {
         new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
     }
@@ -519,10 +516,6 @@ public class RawLocalFileSystem extends 
   
   @Override
   public FileStatus getFileStatus(Path f) throws IOException {
-    if (!Shell.WINDOWS) {
-      return getFileLinkStatusInternal(f, true);
-    }
-
     File path = pathToFile(f);
     if (path.exists()) {
       return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this);
@@ -531,7 +524,6 @@ public class RawLocalFileSystem extends 
     }
   }
 
-  @Deprecated
   static class RawLocalFileStatus extends FileStatus {
     /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
      * We recognize if the information is already loaded by check if
@@ -705,7 +697,6 @@ public class RawLocalFileSystem extends 
    * the given path does not refer to a symlink or there is an error
    * accessing the symlink.
    */
-  @Deprecated
   private String readLink(Path p) {
     /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
      * use getCanonicalPath in File to get the target of the symlink but that
@@ -726,12 +717,7 @@ public class RawLocalFileSystem extends 
    */
   @Override
   public FileStatus getFileLinkStatus(final Path f) throws IOException {
-    FileStatus fi;
-    if (!Shell.WINDOWS) {
-      fi = getFileLinkStatusInternal(f, false);
-    } else {
-      fi = getFileLinkStatusInternal(f);
-    }
+    FileStatus fi = getFileLinkStatusInternal(f);
     // getFileLinkStatus is supposed to return a symlink with a
     // qualified path
     if (fi.isSymlink()) {
@@ -742,12 +728,6 @@ public class RawLocalFileSystem extends 
     return fi;
   }
 
-  /**
-   * Deprecated. Remains for windows support. Should be removed in favor of
-   * {@link #getFileLinkStatusInternal(Path, boolean)} when {@link Stat} gains
-   * support for windows.
-   */
-  @Deprecated
   private FileStatus getFileLinkStatusInternal(final Path f) throws IOException {
     String target = readLink(f);
 
@@ -785,17 +765,9 @@ public class RawLocalFileSystem extends 
     }
   }
 
-  private FileStatus getFileLinkStatusInternal(final Path f,
-      boolean dereference) throws IOException {
-    checkPath(f);
-    Stat stat = new Stat(f, getDefaultBlockSize(f), dereference, this);
-    FileStatus status = stat.getFileStatus();
-    return status;
-  }
-
   @Override
   public Path getLinkTarget(Path f) throws IOException {
-    FileStatus fi = getFileLinkStatusInternal(f, false);
+    FileStatus fi = getFileLinkStatusInternal(f);
     // return an unqualified symlink target
     return fi.getSymlink();
   }

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
Mon Jul 29 23:56:59 2013
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.fs.local;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -26,9 +27,13 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FsConstants;
 import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
 
 /**
  * The RawLocalFs implementation of AbstractFileSystem.
@@ -66,12 +71,108 @@ public class RawLocalFs extends Delegate
   public FsServerDefaults getServerDefaults() throws IOException {
     return LocalConfigKeys.getServerDefaults();
   }
-
+  
   @Override
-  public boolean isValidName(String src) {
-    // Different local file systems have different validation rules. Skip
-    // validation here and just let the OS handle it. This is consistent with
-    // RawLocalFileSystem.
+  public boolean supportsSymlinks() {
     return true;
+  }  
+  
+  @Override
+  public void createSymlink(Path target, Path link, boolean createParent) 
+      throws IOException {
+    final String targetScheme = target.toUri().getScheme();
+    if (targetScheme != null && !"file".equals(targetScheme)) {
+      throw new IOException("Unable to create symlink to non-local file "+
+                            "system: "+target.toString());
+    }
+    if (createParent) {
+      mkdir(link.getParent(), FsPermission.getDirDefault(), true);
+    }
+    // NB: Use createSymbolicLink in java.nio.file.Path once available
+    try {
+      Shell.execCommand(Shell.getSymlinkCommand(
+        Path.getPathWithoutSchemeAndAuthority(target).toString(),
+        Path.getPathWithoutSchemeAndAuthority(link).toString()));
+    } catch (IOException x) {
+      throw new IOException("Unable to create symlink: "+x.getMessage());
+    }
+  }
+
+  /** 
+   * Returns the target of the given symlink. Returns the empty string if  
+   * the given path does not refer to a symlink or there is an error 
+   * acessing the symlink.
+   */
+  private String readLink(Path p) {
+    /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
+     * use getCanonicalPath in File to get the target of the symlink but that 
+     * does not indicate if the given path refers to a symlink.
+     */
+    try {
+      final String path = p.toUri().getPath();
+      return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim(); 
+    } catch (IOException x) {
+      return "";
+    }
+  }
+  
+  /**
+   * Return a FileStatus representing the given path. If the path refers 
+   * to a symlink return a FileStatus representing the link rather than
+   * the object the link refers to.
+   */
+  @Override
+  public FileStatus getFileLinkStatus(final Path f) throws IOException {
+    String target = readLink(f);
+    try {
+      FileStatus fs = getFileStatus(f);
+      // If f refers to a regular file or directory      
+      if (target.isEmpty()) {
+        return fs;
+      }
+      // Otherwise f refers to a symlink
+      return new FileStatus(fs.getLen(), 
+          false,
+          fs.getReplication(), 
+          fs.getBlockSize(),
+          fs.getModificationTime(),
+          fs.getAccessTime(),
+          fs.getPermission(),
+          fs.getOwner(),
+          fs.getGroup(),
+          new Path(target),
+          f);
+    } catch (FileNotFoundException e) {
+      /* The exists method in the File class returns false for dangling 
+       * links so we can get a FileNotFoundException for links that exist.
+       * It's also possible that we raced with a delete of the link. Use
+       * the readBasicFileAttributes method in java.nio.file.attributes 
+       * when available.
+       */
+      if (!target.isEmpty()) {
+        return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(), 
+            "", "", new Path(target), f);        
+      }
+      // f refers to a file or directory that does not exist
+      throw e;
+    }
+  }
+  
+   @Override
+   public boolean isValidName(String src) {
+     // Different local file systems have different validation rules.  Skip
+     // validation here and just let the OS handle it.  This is consistent with
+     // RawLocalFileSystem.
+     return true;
+   }
+  
+  @Override
+  public Path getLinkTarget(Path f) throws IOException {
+    /* We should never get here. Valid local links are resolved transparently
+     * by the underlying local file system and accessing a dangling link will 
+     * result in an IOException, not an UnresolvedLinkException, so FileContext
+     * should never call this function.
+     */
+    throw new AssertionError();
   }
 }

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Mon Jul 29 23:56:59 2013
@@ -58,33 +58,6 @@ abstract public class Shell {
   /** Windows CreateProcess synchronization object */
   public static final Object WindowsProcessLaunchLock = new Object();
 
-  // OSType detection
-
-  public enum OSType {
-    OS_TYPE_LINUX,
-    OS_TYPE_WIN,
-    OS_TYPE_SOLARIS,
-    OS_TYPE_MAC,
-    OS_TYPE_FREEBSD
-  }
-
-  public static final OSType osType = getOSType();
-
-  static private OSType getOSType() {
-    String osName = System.getProperty("os.name");
-    if (Shell.WINDOWS) {
-      return OSType.OS_TYPE_WIN;
-    } else if (osName.contains("SunOS") || osName.contains("Solaris")) {
-      return OSType.OS_TYPE_SOLARIS;
-    } else if (osName.contains("Mac")) {
-      return OSType.OS_TYPE_MAC;
-    } else if (osName.contains("FreeBSD")) {
-      return OSType.OS_TYPE_FREEBSD;
-    } else {
-      return OSType.OS_TYPE_LINUX;
-    }
-  }
-
   /** a Unix command to get the current user's groups list */
   public static String[] getGroupsCommand() {
     return (WINDOWS)? new String[]{"cmd", "/c", "groups"}

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Mon Jul 29 23:56:59 2013
@@ -363,12 +363,12 @@ public class TestLocalFileSystem {
 
     FileStatus status = fileSys.getFileStatus(path);
     assertTrue("check we're actually changing something", newModTime != status.getModificationTime());
-    long accessTime = status.getAccessTime();
+    assertEquals(0, status.getAccessTime());
 
     fileSys.setTimes(path, newModTime, -1);
     status = fileSys.getFileStatus(path);
     assertEquals(newModTime, status.getModificationTime());
-    assertEquals(accessTime, status.getAccessTime());
+    assertEquals(0, status.getAccessTime());
   }
 
   /**

Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java?rev=1508248&r1=1508247&r2=1508248&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestSymlinkLocalFS.java
Mon Jul 29 23:56:59 2013
@@ -30,7 +30,6 @@ import java.net.URI;
 import java.net.URISyntaxException;
 
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.Test;
 
 /**
@@ -124,7 +123,6 @@ abstract public class TestSymlinkLocalFS
     Path fileAbs  = new Path(testBaseDir1()+"/file");
     Path fileQual = new Path(testURI().toString(), fileAbs);
     Path link     = new Path(testBaseDir1()+"/linkToFile");
-    Path linkQual = new Path(testURI().toString(), link.toString());
     wrapper.createSymlink(fileAbs, link, false);
     // Deleting the link using FileContext currently fails because
     // resolve looks up LocalFs rather than RawLocalFs for the path 
@@ -142,15 +140,18 @@ abstract public class TestSymlinkLocalFS
       // Expected. File's exists method returns false for dangling links
     }
     // We can stat a dangling link
-    UserGroupInformation user = UserGroupInformation.getCurrentUser();
     FileStatus fsd = wrapper.getFileLinkStatus(link);
     assertEquals(fileQual, fsd.getSymlink());
     assertTrue(fsd.isSymlink());
     assertFalse(fsd.isDirectory());
-    assertEquals(user.getUserName(), fsd.getOwner());
-    // Compare against user's primary group
-    assertEquals(user.getGroupNames()[0], fsd.getGroup());
-    assertEquals(linkQual, fsd.getPath());
+    assertEquals("", fsd.getOwner());
+    assertEquals("", fsd.getGroup());
+    assertEquals(link, fsd.getPath());
+    assertEquals(0, fsd.getLen());
+    assertEquals(0, fsd.getBlockSize());
+    assertEquals(0, fsd.getReplication());
+    assertEquals(0, fsd.getAccessTime());
+    assertEquals(FsPermission.getDefault(), fsd.getPermission());
     // Accessing the link 
     try {
       readFile(link);



Mime
View raw message