hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1477868 - in /hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common: ./ src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/fs/shell/ src/main/java/org/apache/hadoop/ha/ src/main...
Date Tue, 30 Apr 2013 23:02:37 GMT
Author: szetszwo
Date: Tue Apr 30 23:02:35 2013
New Revision: 1477868

URL: http://svn.apache.org/r1477868
Log:
Merge r1476453 through r1477867 from trunk.

Added:
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java
      - copied unchanged from r1477867, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java
Modified:
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt   (contents, props changed)
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/   (props changed)
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TemporarySocketDirectory.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
    hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt Tue Apr 30 23:02:35 2013
@@ -382,7 +382,7 @@ Trunk (Unreleased)
 
     HADOOP-9190. packaging docs is broken. (Andy Isaacson via atm)
 
-  BREAKDOWN OF HADOOP-8562 SUBTASKS
+  BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
 
     HADOOP-8924. Hadoop Common creating package-info.java must not depend on
     sh. (Chris Nauroth via suresh)
@@ -519,6 +519,21 @@ Trunk (Unreleased)
 
     HADOOP-9443. Port winutils static code analysis change to trunk.
     (Chuan Liu via suresh)
+
+    HADOOP-9290. Some tests cannot load native library on windows.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9500. TestUserGroupInformation#testGetServerSideGroups fails on 
+    Windows due to failure to find winutils.exe. (Chris Nauroth via suresh)
+
+    HADOOP-9490. LocalFileSystem#reportChecksumFailure not closing the 
+    checksum file handle before rename. (Ivan Mitic via suresh)
+
+    HADOOP-9524. Fix ShellCommandFencer to work on Windows.
+    (Arpit Agarwal via suresh)
+
+    HADOOP-9413. Add common utils for File#setReadable/Writable/Executable &
+    File#canRead/Write/Execute that work cross-platform. (Ivan Mitic via suresh)
     
 Release 2.0.5-beta - UNRELEASED
 
@@ -531,6 +546,9 @@ Release 2.0.5-beta - UNRELEASED
     HADOOP-8415. Add getDouble() and setDouble() in
     org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
 
+    HADOOP-9338. FsShell Copy Commands Should Optionally Preserve File
+    Attributes. (Nick White via atm)
+
   IMPROVEMENTS
 
     HADOOP-9253. Capture ulimit info in the logs at service start time.
@@ -562,6 +580,9 @@ Release 2.0.5-beta - UNRELEASED
     HADOOP-9503. Remove sleep between IPC client connect timeouts.
     (Varun Sharma via szetszwo)
 
+    HADOOP-9322. LdapGroupsMapping doesn't seem to set a timeout for
+    its directory search. (harsh)
+
   OPTIMIZATIONS
 
     HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
@@ -644,6 +665,9 @@ Release 2.0.5-beta - UNRELEASED
 
     HADOOP-9473. Typo in FileUtil copy() method. (Glen Mazza via suresh)
 
+    HADOOP-9504. MetricsDynamicMBeanBase has concurrency issues in
+    createMBeanInfo (Liang Xie via jlowe)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -1657,6 +1681,9 @@ Release 0.23.8 - UNRELEASED
     HADOOP-9469. mapreduce/yarn source jars not included in dist tarball
     (Robert Parker via tgraves)
 
+    HADOOP-9504. MetricsDynamicMBeanBase has concurrency issues in
+    createMBeanInfo (Liang Xie via jlowe)
+
 Release 0.23.7 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1476453-1477867

Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1476453-1477867

Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1476453-1477867

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Tue Apr 30 23:02:35 2013
@@ -44,7 +44,6 @@ import org.apache.hadoop.io.nativeio.Nat
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -148,9 +147,9 @@ public class FileUtil {
    * Pure-Java implementation of "chmod +rwx f".
    */
   private static void grantPermissions(final File f) {
-      f.setExecutable(true);
-      f.setReadable(true);
-      f.setWritable(true);
+      FileUtil.setExecutable(f, true);
+      FileUtil.setReadable(f, true);
+      FileUtil.setWritable(f, true);
   }
 
   private static boolean deleteImpl(final File f, final boolean doLog) {
@@ -852,6 +851,129 @@ public class FileUtil {
   }
 
   /**
+   * Platform independent implementation for {@link File#setReadable(boolean)}
+   * File#setReadable does not work as expected on Windows.
+   * @param f input file
+   * @param readable
+   * @return true on success, false otherwise
+   */
+  public static boolean setReadable(File f, boolean readable) {
+    if (Shell.WINDOWS) {
+      try {
+        String permission = readable ? "u+r" : "u-r";
+        FileUtil.chmod(f.getCanonicalPath(), permission, false);
+        return true;
+      } catch (IOException ex) {
+        return false;
+      }
+    } else {
+      return f.setReadable(readable);
+    }
+  }
+
+  /**
+   * Platform independent implementation for {@link File#setWritable(boolean)}
+   * File#setWritable does not work as expected on Windows.
+   * @param f input file
+   * @param writable
+   * @return true on success, false otherwise
+   */
+  public static boolean setWritable(File f, boolean writable) {
+    if (Shell.WINDOWS) {
+      try {
+        String permission = writable ? "u+w" : "u-w";
+        FileUtil.chmod(f.getCanonicalPath(), permission, false);
+        return true;
+      } catch (IOException ex) {
+        return false;
+      }
+    } else {
+      return f.setWritable(writable);
+    }
+  }
+
+  /**
+   * Platform independent implementation for {@link File#setExecutable(boolean)}
+   * File#setExecutable does not work as expected on Windows.
+   * Note: revoking execute permission on folders does not have the same
+   * behavior on Windows as on Unix platforms. Creating, deleting or renaming
+   * a file within that folder will still succeed on Windows.
+   * @param f input file
+   * @param executable
+   * @return true on success, false otherwise
+   */
+  public static boolean setExecutable(File f, boolean executable) {
+    if (Shell.WINDOWS) {
+      try {
+        String permission = executable ? "u+x" : "u-x";
+        FileUtil.chmod(f.getCanonicalPath(), permission, false);
+        return true;
+      } catch (IOException ex) {
+        return false;
+      }
+    } else {
+      return f.setExecutable(executable);
+    }
+  }
+
+  /**
+   * Platform independent implementation for {@link File#canRead()}
+   * @param f input file
+   * @return On Unix, same as {@link File#canRead()}
+   *         On Windows, true if process has read access on the path
+   */
+  public static boolean canRead(File f) {
+    if (Shell.WINDOWS) {
+      try {
+        return NativeIO.Windows.access(f.getCanonicalPath(),
+            NativeIO.Windows.AccessRight.ACCESS_READ);
+      } catch (IOException e) {
+        return false;
+      }
+    } else {
+      return f.canRead();
+    }
+  }
+
+  /**
+   * Platform independent implementation for {@link File#canWrite()}
+   * @param f input file
+   * @return On Unix, same as {@link File#canWrite()}
+   *         On Windows, true if process has write access on the path
+   */
+  public static boolean canWrite(File f) {
+    if (Shell.WINDOWS) {
+      try {
+        return NativeIO.Windows.access(f.getCanonicalPath(),
+            NativeIO.Windows.AccessRight.ACCESS_WRITE);
+      } catch (IOException e) {
+        return false;
+      }
+    } else {
+      return f.canWrite();
+    }
+  }
+
+  /**
+   * Platform independent implementation for {@link File#canExecute()}
+   * @param f input file
+   * @return On Unix, same as {@link File#canExecute()}
+   *         On Windows, true if process has execute access on the path
+   */
+  public static boolean canExecute(File f) {
+    if (Shell.WINDOWS) {
+      try {
+        return NativeIO.Windows.access(f.getCanonicalPath(),
+            NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
+      } catch (IOException e) {
+        return false;
+      }
+    } else {
+      return f.canExecute();
+    }
+  }
+
+  /**
    * Set permissions to the required value. Uses the java primitives instead
    * of forking if group == other.
    * @param f the file to change

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java Tue Apr 30 23:02:35 2013
@@ -103,7 +103,8 @@ public class LocalFileSystem extends Che
       String device = new DF(f, getConf()).getMount();
       File parent = f.getParentFile();
       File dir = null;
-      while (parent!=null && parent.canWrite() && parent.toString().startsWith(device)) {
+      while (parent != null && FileUtil.canWrite(parent) &&
+          parent.toString().startsWith(device)) {
         dir = parent;
         parent = parent.getParentFile();
       }
@@ -130,6 +131,8 @@ public class LocalFileSystem extends Che
       }
       // move checksum file too
       File checkFile = ((RawLocalFileSystem)fs).pathToFile(getChecksumFile(p));
+      // close the stream before rename to release the file handle
+      sums.close();
       b = checkFile.renameTo(new File(badDir, checkFile.getName()+suffix));
       if (!b) {
           LOG.warn("Ignoring failure of renameTo");

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Tue Apr 30 23:02:35 2013
@@ -618,4 +618,27 @@ public class RawLocalFileSystem extends 
         FileUtil.makeShellPath(pathToFile(p), true)));
     }
   }
+ 
+  /**
+   * Sets the {@link Path}'s last modified time <em>only</em> to the given
+   * valid time.
+   *
+   * @param mtime the modification time to set (only if greater than zero).
+   * @param atime currently ignored.
+   * @throws IOException if setting the last modified time fails.
+   */
+  @Override
+  public void setTimes(Path p, long mtime, long atime) throws IOException {
+    File f = pathToFile(p);
+    if(mtime >= 0) {
+      if(!f.setLastModified(mtime)) {
+        throw new IOException(
+          "couldn't set last-modified time to " +
+          mtime +
+          " for " +
+          f.getAbsolutePath());
+      }
+    }
+  }
+
 }

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Tue Apr 30 23:02:35 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.io.IOUtils;
 abstract class CommandWithDestination extends FsCommand {  
   protected PathData dst;
   private boolean overwrite = false;
+  private boolean preserve = false;
   private boolean verifyChecksum = true;
   private boolean writeChecksum = true;
   
@@ -67,6 +68,16 @@ abstract class CommandWithDestination ex
   }
   
   /**
+   * If true, the last modified time, last access time,
+   * owner, group and permission information of the source
+   * file will be preserved as far as target {@link FileSystem}
+   * implementation allows.
+   */
+  protected void setPreserve(boolean preserve) {
+    this.preserve = preserve;
+  }
+
+  /**
    *  The last arg is expected to be a local path, if only one argument is
    *  given then the destination will be the current directory 
    *  @param args is the list of arguments
@@ -227,6 +238,19 @@ abstract class CommandWithDestination ex
     try {
       in = src.fs.open(src.path);
       copyStreamToTarget(in, target);
+      if(preserve) {
+        target.fs.setTimes(
+          target.path,
+          src.stat.getModificationTime(),
+          src.stat.getAccessTime());
+        target.fs.setOwner(
+          target.path,
+          src.stat.getOwner(),
+          src.stat.getGroup());
+        target.fs.setPermission(
+          target.path,
+          src.stat.getPermission());
+      }
     } finally {
       IOUtils.closeStream(in);
     }

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Tue Apr 30 23:02:35 2013
@@ -129,17 +129,19 @@ class CopyCommands {  
 
   static class Cp extends CommandWithDestination {
     public static final String NAME = "cp";
-    public static final String USAGE = "<src> ... <dst>";
+    public static final String USAGE = "[-f] [-p] <src> ... <dst>";
     public static final String DESCRIPTION =
       "Copy files that match the file pattern <src> to a\n" +
       "destination.  When copying multiple files, the destination\n" +
-      "must be a directory.";
+      "must be a directory. Passing -p preserves access and\n" +
+      "modification times, ownership and the mode.\n";
     
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f");
+      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "p");
       cf.parse(args);
       setOverwrite(cf.getOpt("f"));
+      setPreserve(cf.getOpt("p"));
       // should have a -r option
       setRecursive(true);
       getRemoteDestination(args);
@@ -152,20 +154,23 @@ class CopyCommands {  
   public static class Get extends CommandWithDestination {
     public static final String NAME = "get";
     public static final String USAGE =
-      "[-ignoreCrc] [-crc] <src> ... <localdst>";
+      "[-p] [-ignoreCrc] [-crc] <src> ... <localdst>";
     public static final String DESCRIPTION =
       "Copy files that match the file pattern <src>\n" +
       "to the local name.  <src> is kept.  When copying multiple,\n" +
-      "files, the destination must be a directory.";
+      "files, the destination must be a directory. Passing\n" +
+      "-p preserves access and modification times,\n" +
+      "ownership and the mode.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args)
     throws IOException {
       CommandFormat cf = new CommandFormat(
-          1, Integer.MAX_VALUE, "crc", "ignoreCrc");
+          1, Integer.MAX_VALUE, "crc", "ignoreCrc", "p");
       cf.parse(args);
       setWriteChecksum(cf.getOpt("crc"));
       setVerifyChecksum(!cf.getOpt("ignoreCrc"));
+      setPreserve(cf.getOpt("p"));
       setRecursive(true);
       getLocalDestination(args);
     }
@@ -176,16 +181,20 @@ class CopyCommands {  
    */
   public static class Put extends CommandWithDestination {
     public static final String NAME = "put";
-    public static final String USAGE = "<localsrc> ... <dst>";
+    public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>";
     public static final String DESCRIPTION =
       "Copy files from the local file system\n" +
-      "into fs.";
+      "into fs. Copying fails if the file already\n" +
+      "exists, unless the -f flag is given. Passing\n" +
+      "-p preserves access and modification times,\n" +
+      "ownership and the mode.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f");
+      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p");
       cf.parse(args);
       setOverwrite(cf.getOpt("f"));
+      setPreserve(cf.getOpt("p"));
       getRemoteDestination(args);
       // should have a -r option
       setRecursive(true);

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ShellCommandFencer.java Tue Apr 30 23:02:35 2013
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configured;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.util.Shell;
 
 /**
  * Fencing method that runs a shell command. It should be specified
@@ -33,8 +34,8 @@ import com.google.common.annotations.Vis
  * <code>
  *   shell(/path/to/my/script.sh arg1 arg2 ...)
  * </code><br>
- * The string between '(' and ')' is passed directly to a bash shell and
- * may not include any closing parentheses.<p>
+ * The string between '(' and ')' is passed directly to a bash shell
+ * (cmd.exe on Windows) and may not include any closing parentheses.<p>
  * 
  * The shell command will be run with an environment set up to contain
  * all of the current Hadoop configuration variables, with the '_' character 
@@ -58,11 +59,11 @@ public class ShellCommandFencer
 
   /** Prefix for target parameters added to the environment */
   private static final String TARGET_PREFIX = "target_";
-  
+
   @VisibleForTesting
   static Log LOG = LogFactory.getLog(
       ShellCommandFencer.class);
-  
+
   @Override
   public void checkArgs(String args) throws BadFencingConfigurationException {
     if (args == null || args.isEmpty()) {
@@ -74,8 +75,14 @@ public class ShellCommandFencer
 
   @Override
   public boolean tryFence(HAServiceTarget target, String cmd) {
-    ProcessBuilder builder = new ProcessBuilder(
-        "bash", "-e", "-c", cmd);
+    ProcessBuilder builder;
+
+    if (!Shell.WINDOWS) {
+      builder = new ProcessBuilder("bash", "-e", "-c", cmd);
+    } else {
+      builder = new ProcessBuilder("cmd.exe", "/c", cmd);
+    }
+
     setConfAsEnvVars(builder.environment());
     addTargetInfoAsEnvVars(target, builder.environment());
 

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java Tue Apr 30 23:02:35 2013
@@ -356,6 +356,43 @@ public class NativeIO {
     /** Windows only methods used for getOwner() implementation */
     private static native String getOwner(FileDescriptor fd) throws IOException;
 
+    /** Supported list of Windows access right flags */
+    public static enum AccessRight {
+      ACCESS_READ (0x0001),      // FILE_READ_DATA
+      ACCESS_WRITE (0x0002),     // FILE_WRITE_DATA
+      ACCESS_EXECUTE (0x0020);   // FILE_EXECUTE
+
+      private final int accessRight;
+      AccessRight(int access) {
+        accessRight = access;
+      }
+
+      public int accessRight() {
+        return accessRight;
+      }
+    };
+
+    /** Windows only method used to check if the current process has requested
+     *  access rights on the given path. */
+    private static native boolean access0(String path, int requestedAccess);
+
+    /**
+     * Checks whether the current process has desired access rights on
+     * the given path.
+     * 
+     * Longer term this native function can be substituted with JDK7
+     * function Files#isReadable, isWritable, isExecutable.
+     *
+     * @param path input path
+     * @param desiredAccess ACCESS_READ, ACCESS_WRITE or ACCESS_EXECUTE
+     * @return true if access is allowed
+     * @throws IOException I/O exception on error
+     */
+    public static boolean access(String path, AccessRight desiredAccess)
+        throws IOException {
+      return access0(path, desiredAccess.accessRight());
+    }
+
     static {
       if (NativeCodeLoader.isNativeCodeLoaded()) {
         try {

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/util/MetricsDynamicMBeanBase.java Tue Apr 30 23:02:35 2013
@@ -18,9 +18,9 @@
 package org.apache.hadoop.metrics.util;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import javax.management.Attribute;
 import javax.management.AttributeList;
@@ -69,6 +69,7 @@ public abstract class MetricsDynamicMBea
   protected MetricsDynamicMBeanBase(final MetricsRegistry mr, final String aMBeanDescription) {
     metricsRegistry = mr;
     mbeanDescription = aMBeanDescription;
+    metricsRateAttributeMod = new ConcurrentHashMap<String, MetricsBase>();
     createMBeanInfo();
   }
   
@@ -78,7 +79,6 @@ public abstract class MetricsDynamicMBea
   }
   
   private void createMBeanInfo() {
-    metricsRateAttributeMod = new HashMap<String, MetricsBase>();
     boolean needsMinMaxResetOperation = false;
     List<MBeanAttributeInfo> attributesInfo = new ArrayList<MBeanAttributeInfo>();
     MBeanOperationInfo[] operationsInfo = null;

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java Tue Apr 30 23:02:35 2013
@@ -144,7 +144,15 @@ public class LdapGroupsMapping
    */
   public static final String GROUP_NAME_ATTR_KEY = LDAP_CONFIG_PREFIX + ".search.attr.group.name";
   public static final String GROUP_NAME_ATTR_DEFAULT = "cn";
-  
+
+  /*
+   * LDAP {@link SearchControls} attribute to set the time limit
+   * for an invoked directory search. Prevents infinite wait cases.
+   */
+  public static final String DIRECTORY_SEARCH_TIMEOUT =
+    LDAP_CONFIG_PREFIX + ".directory.search.timeout";
+  public static final int DIRECTORY_SEARCH_TIMEOUT_DEFAULT = 10000; // 10s
+
   private static final Log LOG = LogFactory.getLog(LdapGroupsMapping.class);
 
   private static final SearchControls SEARCH_CONTROLS = new SearchControls();
@@ -326,6 +334,9 @@ public class LdapGroupsMapping
     groupNameAttr =
         conf.get(GROUP_NAME_ATTR_KEY, GROUP_NAME_ATTR_DEFAULT);
 
+    int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
+    SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout);
+
     this.conf = conf;
   }
   

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java Tue Apr 30 23:02:35 2013
@@ -23,6 +23,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -160,11 +161,7 @@ public class DiskChecker {
                                    + dir.toString());
     }
 
-    if (Shell.WINDOWS) {
-      checkAccessByFileSystemInteraction(dir);
-    } else {
-      checkAccessByFileMethods(dir);
-    }
+    checkAccessByFileMethods(dir);
   }
 
   /**
@@ -177,68 +174,19 @@ public class DiskChecker {
    */
   private static void checkAccessByFileMethods(File dir)
       throws DiskErrorException {
-    if (!dir.canRead()) {
+    if (!FileUtil.canRead(dir)) {
       throw new DiskErrorException("Directory is not readable: "
                                    + dir.toString());
     }
 
-    if (!dir.canWrite()) {
+    if (!FileUtil.canWrite(dir)) {
       throw new DiskErrorException("Directory is not writable: "
                                    + dir.toString());
     }
 
-    if (!dir.canExecute()) {
+    if (!FileUtil.canExecute(dir)) {
       throw new DiskErrorException("Directory is not executable: "
                                    + dir.toString());
     }
   }
-
-  /**
-   * Checks that the current running process can read, write, and execute the
-   * given directory by attempting each of those operations on the file system.
-   * This method contains several workarounds to known JVM bugs that cause
-   * File.canRead, File.canWrite, and File.canExecute to return incorrect results
-   * on Windows with NTFS ACLs.  See:
-   * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6203387
-   * These bugs are supposed to be fixed in JDK7.
-   * 
-   * @param dir File to check
-   * @throws DiskErrorException if dir is not readable, not writable, or not
-   *   executable
-   */
-  private static void checkAccessByFileSystemInteraction(File dir)
-      throws DiskErrorException {
-    // Make sure we can read the directory by listing it.
-    if (dir.list() == null) {
-      throw new DiskErrorException("Directory is not readable: "
-                                   + dir.toString());
-    }
-
-    // Make sure we can write to the directory by creating a temp file in it.
-    try {
-      File tempFile = File.createTempFile("checkDirAccess", null, dir);
-      if (!tempFile.delete()) {
-        throw new DiskErrorException("Directory is not writable: "
-                                     + dir.toString());
-      }
-    } catch (IOException e) {
-      throw new DiskErrorException("Directory is not writable: "
-                                   + dir.toString(), e);
-    }
-
-    // Make sure the directory is executable by trying to cd into it.  This
-    // launches a separate process.  It does not change the working directory of
-    // the current process.
-    try {
-      String[] cdCmd = new String[] { "cmd", "/C", "cd",
-          dir.getAbsolutePath() };
-      Shell.execCommand(null, cdCmd, SHELL_TIMEOUT);
-    } catch (Shell.ExitCodeException e) {
-      throw new DiskErrorException("Directory is not executable: "
-                                   + dir.toString(), e);
-    } catch (IOException e) {
-      throw new DiskErrorException("Directory is not executable: "
-                                   + dir.toString(), e);
-    }
-  }
 }

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c Tue Apr 30 23:02:35 2013
@@ -812,6 +812,42 @@ cleanup:
 #endif
 }
 
+/*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
+ * Method:    access0
+ * Signature: (Ljava/lang/String;I)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_access0
+  (JNIEnv *env, jclass clazz, jstring jpath, jint jaccess)
+{
+#ifdef UNIX
+  THROW(env, "java/io/IOException",
+    "The function access0(path, access) is not supported on Unix");
+  return NULL;
+#endif
+
+#ifdef WINDOWS
+  LPCWSTR path = NULL;
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  ACCESS_MASK access = (ACCESS_MASK)jaccess;
+  BOOL allowed = FALSE;
+
+  path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+  if (!path) goto cleanup; // exception was thrown
+
+  dwRtnCode = CheckAccessForCurrentUser(path, access, &allowed);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    throw_ioe(env, dwRtnCode);
+    goto cleanup;
+  }
+
+cleanup:
+  if (path) (*env)->ReleaseStringChars(env, jpath, path);
+
+  return (jboolean)allowed;
+#endif
+}
+
 JNIEXPORT void JNICALL 
 Java_org_apache_hadoop_io_nativeio_NativeIO_renameTo0(JNIEnv *env, 
 jclass clazz, jstring jsrc, jstring jdst)

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Tue Apr 30 23:02:35 2013
@@ -213,6 +213,17 @@
 </property>
 
 <property>
+  <name>hadoop.security.group.mapping.ldap.directory.search.timeout</name>
+  <value>10000</value>
+  <description>
+    The attribute applied to the LDAP SearchControl properties to set a
+    maximum time limit when searching and awaiting a result.
+    Set to 0 if infinite wait period is desired.
+    Default is 10 seconds. Units in milliseconds.
+  </description>
+</property>
+
+<property>
   <name>hadoop.security.service.user.name.key</name>
   <value></value>
   <description>

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h Tue Apr 30 23:02:35 2013
@@ -110,6 +110,11 @@ void SystemInfoUsage();
 DWORD GetFileInformationByName(__in LPCWSTR pathName,  __in BOOL followLink,
   __out LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
 
+DWORD CheckAccessForCurrentUser(
+  __in PCWSTR pathName,
+  __in ACCESS_MASK requestedAccess,
+  __out BOOL *allowed);
+
 DWORD ConvertToLongPath(__in PCWSTR path, __deref_out PWSTR *newPath);
 
 DWORD GetSidFromAcctNameW(__in PCWSTR acctName, __out PSID* ppSid);

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c Tue Apr 30 23:02:35 2013
@@ -567,7 +567,7 @@ static DWORD GetEffectiveRightsForSid(PS
   PSID pSid,
   PACCESS_MASK pAccessRights)
 {
-  AUTHZ_RESOURCE_MANAGER_HANDLE hManager;
+  AUTHZ_RESOURCE_MANAGER_HANDLE hManager = NULL;
   LUID unusedId = { 0 };
   AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClientContext = NULL;
   DWORD dwRtnCode = ERROR_SUCCESS;
@@ -581,6 +581,10 @@ static DWORD GetEffectiveRightsForSid(PS
     return GetLastError();
   }
 
+  // Pass AUTHZ_SKIP_TOKEN_GROUPS to the function to avoid querying user group
+  // information for access check. This allows us to model POSIX permissions
+  // on Windows, where a user can have less permissions than a group it
+  // belongs to.
   if(!AuthzInitializeContextFromSid(AUTHZ_SKIP_TOKEN_GROUPS,
     pSid, hManager, NULL, unusedId, NULL, &hAuthzClientContext))
   {
@@ -594,17 +598,116 @@ static DWORD GetEffectiveRightsForSid(PS
     ret = dwRtnCode;
     goto GetEffectiveRightsForSidEnd;
   }
-  if (!AuthzFreeContext(hAuthzClientContext))
+
+GetEffectiveRightsForSidEnd:
+  if (hManager != NULL)
   {
-    ret = GetLastError();
-    goto GetEffectiveRightsForSidEnd;
+    (void)AuthzFreeResourceManager(hManager);
+  }
+  if (hAuthzClientContext != NULL)
+  {
+    (void)AuthzFreeContext(hAuthzClientContext);
   }
 
-GetEffectiveRightsForSidEnd:
   return ret;
 }
 
 //----------------------------------------------------------------------------
+// Function: CheckAccessForCurrentUser
+//
+// Description:
+//   Checks if the current process has the requested access rights on the given
+//   path. Based on the following MSDN article:
+//   http://msdn.microsoft.com/en-us/library/windows/desktop/ff394771(v=vs.85).aspx
+//
+// Returns:
+//   ERROR_SUCCESS: on success
+//
+DWORD CheckAccessForCurrentUser(
+  __in PCWSTR pathName,
+  __in ACCESS_MASK requestedAccess,
+  __out BOOL *allowed)
+{
+  DWORD dwRtnCode = ERROR_SUCCESS;
+
+  LPWSTR longPathName = NULL;
+  HANDLE hProcessToken = NULL;
+  PSECURITY_DESCRIPTOR pSd = NULL;
+
+  AUTHZ_RESOURCE_MANAGER_HANDLE hManager = NULL;
+  AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClientContext = NULL;
+  LUID Luid = {0, 0};
+
+  ACCESS_MASK currentUserAccessRights = 0;
+
+  // Prepend the long path prefix if needed
+  dwRtnCode = ConvertToLongPath(pathName, &longPathName);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    goto CheckAccessEnd;
+  }
+
+  // Get SD of the given path. OWNER and DACL security info must be
+  // requested, otherwise, AuthzAccessCheck fails with invalid parameter
+  // error.
+  dwRtnCode = GetNamedSecurityInfo(longPathName, SE_FILE_OBJECT,
+    OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION |
+    DACL_SECURITY_INFORMATION,
+    NULL, NULL, NULL, NULL, &pSd);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    goto CheckAccessEnd;
+  }
+
+  // Get current process token
+  if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &hProcessToken))
+  {
+    dwRtnCode = GetLastError();
+    goto CheckAccessEnd;
+  }
+
+  if (!AuthzInitializeResourceManager(AUTHZ_RM_FLAG_NO_AUDIT, NULL, NULL,
+    NULL, NULL, &hManager))
+  {
+    dwRtnCode = GetLastError();
+    goto CheckAccessEnd;
+  }
+
+  if(!AuthzInitializeContextFromToken(0, hProcessToken, hManager, NULL,
+    Luid, NULL, &hAuthzClientContext))
+  {
+    dwRtnCode = GetLastError();
+    goto CheckAccessEnd;
+  }
+
+  dwRtnCode = GetAccess(hAuthzClientContext, pSd, &currentUserAccessRights);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    goto CheckAccessEnd;
+  }
+
+  *allowed = ((currentUserAccessRights & requestedAccess) == requestedAccess);
+
+CheckAccessEnd:
+  LocalFree(longPathName);
+  LocalFree(pSd);
+  if (hProcessToken != NULL)
+  {
+    CloseHandle(hProcessToken);
+  }
+  if (hManager != NULL)
+  {
+    (void)AuthzFreeResourceManager(hManager);
+  }
+  if (hAuthzClientContext != NULL)
+  {
+    (void)AuthzFreeContext(hAuthzClientContext);
+  }
+
+  return dwRtnCode;
+}
+
+//----------------------------------------------------------------------------
 // Function: FindFileOwnerAndPermission
 //
 // Description:

Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1476453-1477867

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java Tue Apr 30 23:02:35 2013
@@ -353,15 +353,15 @@ public class TestFileUtil {
   }
   
   private static void grantPermissions(final File f) {
-    f.setReadable(true);
-    f.setWritable(true);
-    f.setExecutable(true);
+    FileUtil.setReadable(f, true);
+    FileUtil.setWritable(f, true);
+    FileUtil.setExecutable(f, true);
   }
   
   private static void revokePermissions(final File f) {
-     f.setWritable(false);
-     f.setExecutable(false);
-     f.setReadable(false);
+     FileUtil.setWritable(f, false);
+     FileUtil.setExecutable(f, false);
+     FileUtil.setReadable(f, false);
   }
   
   // Validates the return value.

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java Tue Apr 30 23:02:35 2013
@@ -52,14 +52,15 @@ public class TestLocalFileSystem {
   
   @Before
   public void setup() throws IOException {
-    conf = new Configuration();
+    conf = new Configuration(false);
+    conf.set("fs.file.impl", LocalFileSystem.class.getName());
     fileSys = FileSystem.getLocal(conf);
     fileSys.delete(new Path(TEST_ROOT_DIR), true);
   }
   
   @After
   public void after() throws IOException {
-    base.setWritable(true);
+    FileUtil.setWritable(base, true);
     FileUtil.fullyDelete(base);
     assertTrue(!base.exists());
   }
@@ -67,7 +68,7 @@ public class TestLocalFileSystem {
   /**
    * Test the capability of setting the working directory.
    */
-  @Test
+  @Test(timeout = 1000)
   public void testWorkingDirectory() throws IOException {
     Path origDir = fileSys.getWorkingDirectory();
     Path subdir = new Path(TEST_ROOT_DIR, "new");
@@ -121,10 +122,9 @@ public class TestLocalFileSystem {
    * test Syncable interface on raw local file system
    * @throws IOException
    */
-  @Test
+  @Test(timeout = 1000)
   public void testSyncable() throws IOException {
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.getLocal(conf).getRawFileSystem();
+    FileSystem fs = fileSys.getRawFileSystem();
     Path file = new Path(TEST_ROOT_DIR, "syncable");
     FSDataOutputStream out = fs.create(file);;
     final int bytesWritten = 1;
@@ -155,76 +155,68 @@ public class TestLocalFileSystem {
     }
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testCopy() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path src = new Path(TEST_ROOT_DIR, "dingo");
     Path dst = new Path(TEST_ROOT_DIR, "yak");
-    writeFile(fs, src, 1);
-    assertTrue(FileUtil.copy(fs, src, fs, dst, true, false, conf));
-    assertTrue(!fs.exists(src) && fs.exists(dst));
-    assertTrue(FileUtil.copy(fs, dst, fs, src, false, false, conf));
-    assertTrue(fs.exists(src) && fs.exists(dst));
-    assertTrue(FileUtil.copy(fs, src, fs, dst, true, true, conf));
-    assertTrue(!fs.exists(src) && fs.exists(dst));
-    fs.mkdirs(src);
-    assertTrue(FileUtil.copy(fs, dst, fs, src, false, false, conf));
+    writeFile(fileSys, src, 1);
+    assertTrue(FileUtil.copy(fileSys, src, fileSys, dst, true, false, conf));
+    assertTrue(!fileSys.exists(src) && fileSys.exists(dst));
+    assertTrue(FileUtil.copy(fileSys, dst, fileSys, src, false, false, conf));
+    assertTrue(fileSys.exists(src) && fileSys.exists(dst));
+    assertTrue(FileUtil.copy(fileSys, src, fileSys, dst, true, true, conf));
+    assertTrue(!fileSys.exists(src) && fileSys.exists(dst));
+    fileSys.mkdirs(src);
+    assertTrue(FileUtil.copy(fileSys, dst, fileSys, src, false, false, conf));
     Path tmp = new Path(src, dst.getName());
-    assertTrue(fs.exists(tmp) && fs.exists(dst));
-    assertTrue(FileUtil.copy(fs, dst, fs, src, false, true, conf));
-    assertTrue(fs.delete(tmp, true));
-    fs.mkdirs(tmp);
+    assertTrue(fileSys.exists(tmp) && fileSys.exists(dst));
+    assertTrue(FileUtil.copy(fileSys, dst, fileSys, src, false, true, conf));
+    assertTrue(fileSys.delete(tmp, true));
+    fileSys.mkdirs(tmp);
     try {
-      FileUtil.copy(fs, dst, fs, src, true, true, conf);
+      FileUtil.copy(fileSys, dst, fileSys, src, true, true, conf);
       fail("Failed to detect existing dir");
     } catch (IOException e) {
       // Expected
     }
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testHomeDirectory() throws IOException {
-    Configuration conf = new Configuration();
-    FileSystem fileSys = FileSystem.getLocal(conf);
     Path home = new Path(System.getProperty("user.home"))
       .makeQualified(fileSys);
     Path fsHome = fileSys.getHomeDirectory();
     assertEquals(home, fsHome);
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testPathEscapes() throws IOException {
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.getLocal(conf);
     Path path = new Path(TEST_ROOT_DIR, "foo%bar");
-    writeFile(fs, path, 1);
-    FileStatus status = fs.getFileStatus(path);
-    assertEquals(path.makeQualified(fs), status.getPath());
-    cleanupFile(fs, path);
+    writeFile(fileSys, path, 1);
+    FileStatus status = fileSys.getFileStatus(path);
+    assertEquals(path.makeQualified(fileSys), status.getPath());
+    cleanupFile(fileSys, path);
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testMkdirs() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path test_dir = new Path(TEST_ROOT_DIR, "test_dir");
     Path test_file = new Path(TEST_ROOT_DIR, "file1");
-    assertTrue(fs.mkdirs(test_dir));
+    assertTrue(fileSys.mkdirs(test_dir));
    
-    writeFile(fs, test_file, 1);
+    writeFile(fileSys, test_file, 1);
     // creating dir over a file
     Path bad_dir = new Path(test_file, "another_dir");
     
     try {
-      fs.mkdirs(bad_dir);
+      fileSys.mkdirs(bad_dir);
       fail("Failed to detect existing file in path");
     } catch (FileAlreadyExistsException e) { 
       // Expected
     }
     
     try {
-      fs.mkdirs(null);
+        fileSys.mkdirs(null);
       fail("Failed to detect null in mkdir arg");
     } catch (IllegalArgumentException e) {
       // Expected
@@ -232,26 +224,23 @@ public class TestLocalFileSystem {
   }
 
   /** Test deleting a file, directory, and non-existent path */
-  @Test
+  @Test(timeout = 1000)
   public void testBasicDelete() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path dir1 = new Path(TEST_ROOT_DIR, "dir1");
     Path file1 = new Path(TEST_ROOT_DIR, "file1");
     Path file2 = new Path(TEST_ROOT_DIR+"/dir1", "file2");
     Path file3 = new Path(TEST_ROOT_DIR, "does-not-exist");
-    assertTrue(fs.mkdirs(dir1));
-    writeFile(fs, file1, 1);
-    writeFile(fs, file2, 1);
+    assertTrue(fileSys.mkdirs(dir1));
+    writeFile(fileSys, file1, 1);
+    writeFile(fileSys, file2, 1);
     assertFalse("Returned true deleting non-existant path", 
-        fs.delete(file3));
-    assertTrue("Did not delete file", fs.delete(file1));
-    assertTrue("Did not delete non-empty dir", fs.delete(dir1));
+            fileSys.delete(file3));
+    assertTrue("Did not delete file", fileSys.delete(file1));
+    assertTrue("Did not delete non-empty dir", fileSys.delete(dir1));
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testStatistics() throws Exception {
-    FileSystem.getLocal(new Configuration());
     int fileSchemeCount = 0;
     for (Statistics stats : FileSystem.getAllStatistics()) {
       if (stats.getScheme().equals("file")) {
@@ -261,12 +250,10 @@ public class TestLocalFileSystem {
     assertEquals(1, fileSchemeCount);
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testHasFileDescriptor() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path path = new Path(TEST_ROOT_DIR, "test-file");
-    writeFile(fs, path, 1);
+    writeFile(fileSys, path, 1);
     BufferedFSInputStream bis = null;
     try {
       bis = new BufferedFSInputStream(new RawLocalFileSystem()
@@ -277,20 +264,18 @@ public class TestLocalFileSystem {
     }
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testListStatusWithColons() throws IOException {
     assumeTrue(!Shell.WINDOWS);
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     File colonFile = new File(TEST_ROOT_DIR, "foo:bar");
     colonFile.mkdirs();
-    FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR));
+    FileStatus[] stats = fileSys.listStatus(new Path(TEST_ROOT_DIR));
     assertEquals("Unexpected number of stats", 1, stats.length);
     assertEquals("Bad path from stat", colonFile.getAbsolutePath(),
         stats[0].getPath().toUri().getPath());
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testReportChecksumFailure() throws IOException {
     base.mkdirs();
     assertTrue(base.exists() && base.isDirectory());
@@ -298,7 +283,7 @@ public class TestLocalFileSystem {
     final File dir1 = new File(base, "dir1");
     final File dir2 = new File(dir1, "dir2");
     dir2.mkdirs();
-    assertTrue(dir2.exists() && dir2.canWrite());
+    assertTrue(dir2.exists() && FileUtil.canWrite(dir2));
     
     final String dataFileName = "corruptedData";
     final Path dataPath = new Path(new File(dir2, dataFileName).toURI());
@@ -321,7 +306,7 @@ public class TestLocalFileSystem {
     // this is a hack to force the #reportChecksumFailure() method to stop
     // climbing up at the 'base' directory and use 'dir1/bad_files' as the 
     // corrupted files storage:
-    base.setWritable(false);
+    FileUtil.setWritable(base, false);
     
     FSDataInputStream dataFsdis = fileSys.open(dataPath);
     FSDataInputStream checksumFsdis = fileSys.open(checksumPath);
@@ -363,4 +348,23 @@ public class TestLocalFileSystem {
     assertTrue(checksumFileFound);
   }
   
+  @Test(timeout = 1000)
+  public void testSetTimes() throws Exception {
+    Path path = new Path(TEST_ROOT_DIR, "set-times");
+    writeFile(fileSys, path, 1);
+
+    // test only to the nearest second, as the raw FS may not
+    // support millisecond timestamps
+    long newModTime = 12345000;
+
+    FileStatus status = fileSys.getFileStatus(path);
+    assertTrue("check we're actually changing something", newModTime != status.getModificationTime());
+    assertEquals(0, status.getAccessTime());
+
+    fileSys.setTimes(path, newModTime, -1);
+    status = fileSys.getFileStatus(path);
+    assertEquals(newModTime, status.getModificationTime());
+    assertEquals(0, status.getAccessTime());
+}
+
 }

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestNodeFencer.java Tue Apr 30 23:02:35 2013
@@ -24,6 +24,7 @@ import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.Shell;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -33,7 +34,12 @@ import com.google.common.collect.Lists;
 public class TestNodeFencer {
 
   private HAServiceTarget MOCK_TARGET;
-  
+
+  // Fencer shell commands that always return true on Unix and Windows
+  // respectively. Lacking the POSIX 'true' command on Windows, we use
+  // the batch command 'rem'.
+  private static String FENCER_TRUE_COMMAND_UNIX = "shell(true)";
+  private static String FENCER_TRUE_COMMAND_WINDOWS = "shell(rem)";
 
   @Before
   public void clearMockState() {
@@ -48,6 +54,11 @@ public class TestNodeFencer {
         .when(MOCK_TARGET).getAddress();
   }
 
+  private static String getFencerTrueCommand() {
+    return Shell.WINDOWS ?
+        FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
+  }
+
   @Test
   public void testSingleFencer() throws BadFencingConfigurationException {
     NodeFencer fencer = setupFencer(
@@ -100,7 +111,7 @@ public class TestNodeFencer {
 
   @Test
   public void testShortNameShell() throws BadFencingConfigurationException {
-    NodeFencer fencer = setupFencer("shell(true)");
+    NodeFencer fencer = setupFencer(getFencerTrueCommand());
     assertTrue(fencer.fence(MOCK_TARGET));
   }
 

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java Tue Apr 30 23:02:35 2013
@@ -240,6 +240,44 @@ public class TestNativeIO {
 
   }
 
+  /** Validate access checks on Windows */
+  @Test (timeout = 30000)
+  public void testAccess() throws Exception {
+    if (!Path.WINDOWS) {
+      return;
+    }
+
+    File testFile = new File(TEST_DIR, "testfileaccess");
+    assertTrue(testFile.createNewFile());
+
+    // Validate ACCESS_READ
+    FileUtil.setReadable(testFile, false);
+    assertFalse(NativeIO.Windows.access(testFile.getAbsolutePath(),
+        NativeIO.Windows.AccessRight.ACCESS_READ));
+
+    FileUtil.setReadable(testFile, true);
+    assertTrue(NativeIO.Windows.access(testFile.getAbsolutePath(),
+        NativeIO.Windows.AccessRight.ACCESS_READ));
+
+    // Validate ACCESS_WRITE
+    FileUtil.setWritable(testFile, false);
+    assertFalse(NativeIO.Windows.access(testFile.getAbsolutePath(),
+        NativeIO.Windows.AccessRight.ACCESS_WRITE));
+
+    FileUtil.setWritable(testFile, true);
+    assertTrue(NativeIO.Windows.access(testFile.getAbsolutePath(),
+        NativeIO.Windows.AccessRight.ACCESS_WRITE));
+
+    // Validate ACCESS_EXECUTE
+    FileUtil.setExecutable(testFile, false);
+    assertFalse(NativeIO.Windows.access(testFile.getAbsolutePath(),
+        NativeIO.Windows.AccessRight.ACCESS_EXECUTE));
+
+    FileUtil.setExecutable(testFile, true);
+    assertTrue(NativeIO.Windows.access(testFile.getAbsolutePath(),
+        NativeIO.Windows.AccessRight.ACCESS_EXECUTE));
+  }
+
   @Test (timeout = 30000)
   public void testOpenMissingWithoutCreate() throws Exception {
     if (Path.WINDOWS) {

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TemporarySocketDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TemporarySocketDirectory.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TemporarySocketDirectory.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TemporarySocketDirectory.java Tue Apr 30 23:02:35 2013
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.util.Random;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileUtil;
 
 /**
  * Create a temporary directory in which sockets can be created.
@@ -37,7 +38,7 @@ public class TemporarySocketDirectory im
     dir = new File(tmp, "socks." + (System.currentTimeMillis() +
         "." + (new Random().nextInt())));
     dir.mkdirs();
-    dir.setWritable(true, true);
+    FileUtil.setWritable(dir, true);
   }
 
   public File getDir() {

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java Tue Apr 30 23:02:35 2013
@@ -76,7 +76,9 @@ public class TestUserGroupInformation {
     javax.security.auth.login.Configuration.setConfiguration(
         new DummyLoginConfiguration());
     // doesn't matter what it is, but getGroups needs it set...
-    System.setProperty("hadoop.home.dir", "/tmp");
+    // use HADOOP_HOME environment variable to prevent interfering with logic
+    // that finds winutils.exe
+    System.setProperty("hadoop.home.dir", System.getenv("HADOOP_HOME"));
     // fake the realm is kerberos is enabled
     System.setProperty("java.security.krb5.kdc", "");
     System.setProperty("java.security.krb5.realm", "DEFAULT.REALM");

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java Tue Apr 30 23:02:35 2013
@@ -28,6 +28,8 @@ import java.lang.management.ManagementFa
 import java.lang.management.ThreadInfo;
 import java.lang.management.ThreadMXBean;
 
+import org.apache.hadoop.fs.FileUtil;
+
 public class TestShell extends TestCase {
 
   private static class Command extends Shell {
@@ -92,7 +94,7 @@ public class TestShell extends TestCase 
     PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
     writer.println(timeoutCommand);
     writer.close();
-    shellFile.setExecutable(true);
+    FileUtil.setExecutable(shellFile, true);
     Shell.ShellCommandExecutor shexc 
     = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
                                       null, null, 100);

Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml?rev=1477868&r1=1477867&r2=1477868&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml Tue Apr 30 23:02:35 2013
@@ -133,7 +133,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -141,7 +141,15 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*files, the destination must be a directory.( )*</expected-output>
+          <expected-output>^( |\t)*files, the destination must be a directory.( )*Passing( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -276,7 +284,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-cp &lt;src&gt; \.\.\. &lt;dst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt; to a( )*</expected-output>
+          <expected-output>^-cp \[-f\] \[-p\] &lt;src&gt; \.\.\. &lt;dst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt; to a( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -284,7 +292,11 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*must be a directory.( )*</expected-output>
+          <expected-output>^( |\t)*must be a directory.( )*Passing -p preserves access and( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*modification times, ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -372,11 +384,23 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-put &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Copy files from the local file system</expected-output>
+          <expected-output>^-put \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Copy files from the local file system</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*into fs.( )*Copying fails if the file already( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*into fs.( )*</expected-output>
+          <expected-output>^( |\t)*exists, unless the -f flag is given.( )*Passing( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -391,7 +415,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyFromLocal &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Identical to the -put command\.</expected-output>
+          <expected-output>^-copyFromLocal \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Identical to the -put command\.</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -426,7 +450,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -434,7 +458,15 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*files, the destination must be a directory.( )*</expected-output>
+          <expected-output>^( |\t)*files, the destination must be a directory.( )*Passing( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -512,7 +544,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyToLocal \[-ignoreCrc\] \[-crc\] &lt;src&gt; \.\.\. &lt;localdst&gt;:\s+Identical to the -get command.</expected-output>
+          <expected-output>^-copyToLocal \[-p\] \[-ignoreCrc\] \[-crc\] &lt;src&gt; \.\.\. &lt;localdst&gt;:\s+Identical to the -get command.</expected-output>
         </comparator>
       </comparators>
     </test>



Mime
View raw message