hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1422280 [1/2] - in /hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common: ./ src/main/bin/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/io/nativeio/ src/main/java/org/apache/hadoop/security/ sr...
Date Sat, 15 Dec 2012 16:29:50 GMT
Author: suresh
Date: Sat Dec 15 16:29:47 2012
New Revision: 1422280

URL: http://svn.apache.org/viewvc?rev=1422280&view=rev
Log:
HADOOP-9081. Add TestWinUtils. Contributed by Chuan Liu, Ivan Mitic, Chris Nauroth, and Bikas Saha.

Added:
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar   (with props)
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz   (with props)
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java
Removed:
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/tests/test-all.bat
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/tests/test-winutils-basics.bat
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/tests/test-winutils-chmod.bat
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/tests/test-winutils-chown.bat
Modified:
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chown.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/groups.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/ls.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/main.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
    hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt Sat Dec 15 16:29:47 2012
@@ -66,3 +66,6 @@ branch-trunk-win changes - unreleased
   HADOOP-9144. Fix findbugs warnings. (Chris Nauroth via suresh)
 
   HADOOP-8981. TestMetricsSystemImpl fails on Windows. (Xuan Gong via suresh)
+
+  HADOOP-9081. Add TestWinUtils. (Chuan Liu, Ivan Mitic, Chris Nauroth, 
+  and Bikas Saha via suresh)

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml Sat Dec 15 16:29:47 2012
@@ -241,6 +241,11 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>1.4</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -331,6 +336,23 @@
             </configuration>
           </execution>
           <execution>
+            <id>copy-test-tarballs</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <copy toDir="${test.cache.data}">
+                  <fileset dir="${basedir}/src/test/java/org/apache/hadoop/fs">
+                    <include name="test-untar.tar"/>
+                    <include name="test-untar.tgz"/>
+                  </fileset>
+                </copy>
+              </target>
+            </configuration>
+          </execution>
+          <execution>
             <phase>pre-site</phase>
             <goals>
               <goal>run</goal>

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd Sat Dec 15 16:29:47 2012
@@ -173,10 +173,15 @@ call :updatepath %HADOOP_BIN_PATH%
 
 :updatepath
   set path_to_add=%*
-  set current_path_comparable=%path:(x86)=%
+  set current_path_comparable=%path%
   set current_path_comparable=%current_path_comparable: =_%
-  set path_to_add_comparable=%path_to_add:(x86)=%
+  set current_path_comparable=%current_path_comparable:(=_%
+  set current_path_comparable=%current_path_comparable:)=_%
+  set path_to_add_comparable=%path_to_add%
   set path_to_add_comparable=%path_to_add_comparable: =_%
+  set path_to_add_comparable=%path_to_add_comparable:(=_%
+  set path_to_add_comparable=%path_to_add_comparable:)=_%
+
   for %%i in ( %current_path_comparable% ) do (
     if /i "%%i" == "%path_to_add_comparable%" (
       set path_to_add_exist=true

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Sat Dec 15 16:29:47 2012
@@ -21,9 +21,12 @@ package org.apache.hadoop.fs;
 import java.io.*;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -538,33 +541,46 @@ public class FileUtil {
    * @throws IOException
    */
   public static void unTar(File inFile, File untarDir) throws IOException {
-    if (!untarDir.mkdirs()) {           
+    if (!untarDir.mkdirs()) {
       if (!untarDir.isDirectory()) {
         throw new IOException("Mkdirs failed to create " + untarDir);
       }
     }
 
-    StringBuilder untarCommand = new StringBuilder();
     boolean gzipped = inFile.toString().endsWith("gz");
+    if(Shell.WINDOWS) {
+      // Tar is not native to Windows. Use simple Java based implementation for 
+      // tests and simple tar archives
+      unTarUsingJava(inFile, untarDir, gzipped);
+    }
+    else {
+      // spawn tar utility to untar archive for full fledged unix behavior such 
+      // as resolving symlinks in tar archives
+      unTarUsingTar(inFile, untarDir, gzipped);
+    }
+  }
+  
+  private static void unTarUsingTar(File inFile, File untarDir,
+      boolean gzipped) throws IOException {
+    StringBuffer untarCommand = new StringBuffer();
     if (gzipped) {
-      untarCommand.append((Shell.WINDOWS) ? " gzip -dc \"" : " gzip -dc '");
+      untarCommand.append(" gzip -dc '");
       untarCommand.append(FileUtil.makeShellPath(inFile));
-      untarCommand.append((Shell.WINDOWS) ? "\" | (" : "' | (");
+      untarCommand.append("' | (");
     } 
-    untarCommand.append((Shell.WINDOWS) ? "cd \"" : "cd '");
+    untarCommand.append("cd '");
     untarCommand.append(FileUtil.makeShellPath(untarDir)); 
-    untarCommand.append((Shell.WINDOWS) ? "\" & " : "' ; ");
+    untarCommand.append("' ; ");
 
     // Force the archive path as local on Windows as it can have a colon
-    untarCommand.append((Shell.WINDOWS) ? "tar --force-local -xf " : "tar -xf ");
+    untarCommand.append("tar -xf ");
 
     if (gzipped) {
       untarCommand.append(" -)");
     } else {
       untarCommand.append(FileUtil.makeShellPath(inFile));
     }
-    String[] shellCmd = {(Shell.WINDOWS)?"cmd":"bash", (Shell.WINDOWS)?"/c":"-c",
-      untarCommand.toString() };
+    String[] shellCmd = { "bash", "-c", untarCommand.toString() };
     ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
     shexec.execute();
     int exitcode = shexec.getExitCode();
@@ -573,7 +589,62 @@ public class FileUtil {
                   ". Tar process exited with exit code " + exitcode);
     }
   }
+  
+  private static void unTarUsingJava(File inFile, File untarDir,
+      boolean gzipped) throws IOException {
+    InputStream inputStream = null;
+    if (gzipped) {
+      inputStream = new BufferedInputStream(new GZIPInputStream(
+          new FileInputStream(inFile)));
+    } else {
+      inputStream = new BufferedInputStream(new FileInputStream(inFile));
+    }
+
+    TarArchiveInputStream tis = new TarArchiveInputStream(inputStream);
+
+    for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
+      unpackEntries(tis, entry, untarDir);
+      entry = tis.getNextTarEntry();
+    }
+  }
+  
+  private static void unpackEntries(TarArchiveInputStream tis,
+      TarArchiveEntry entry, File outputDir) throws IOException {
+    if (entry.isDirectory()) {
+      File subDir = new File(outputDir, entry.getName());
+      if (!subDir.mkdir() && !subDir.isDirectory()) {
+        throw new IOException("Mkdirs failed to create tar internal dir "
+            + outputDir);
+      }
+
+      for (TarArchiveEntry e : entry.getDirectoryEntries()) {
+        unpackEntries(tis, e, subDir);
+      }
+
+      return;
+    }
 
+    File outputFile = new File(outputDir, entry.getName());
+    if (!outputDir.exists()) {
+      if (!outputDir.mkdirs()) {
+        throw new IOException("Mkdirs failed to create tar internal dir "
+            + outputDir);
+      }
+    }
+
+    int count;
+    byte data[] = new byte[2048];
+    BufferedOutputStream outputStream = new BufferedOutputStream(
+        new FileOutputStream(outputFile));
+
+    while ((count = tis.read(data)) != -1) {
+      outputStream.write(data, 0, count);
+    }
+
+    outputStream.flush();
+    outputStream.close();
+  }
+  
   /**
    * Class for creating hardlinks.
    * Supports Unix, Cygwin, WindXP.
@@ -598,11 +669,34 @@ public class FileUtil {
    */
   public static int symLink(String target, String linkname) throws IOException{
     // Run the input paths through Java's File so that they are converted to the
-    // native OS form. FIXME: Long term fix is to expose symLink API that
-    // accepts File instead of String, as symlinks can only be created on the
-    // local FS.
-    String[] cmd = Shell.getSymlinkCommand(new File(target).getPath(),
-        new File(linkname).getPath());
+    // native OS form
+    File targetFile = new File(target);
+    File linkFile = new File(linkname);
+
+    // If not on Java7+, copy a file instead of creating a symlink since
+    // Java6 has close to no support for symlinks on Windows. Specifically
+    // File#length and File#renameTo do not work as expected.
+    // (see HADOOP-9061 for additional details)
+    // We still create symlinks for directories, since the scenario in this
+    // case is different. The directory content could change in which
+    // case the symlink loses its purpose (for example task attempt log folder
+    // is symlinked under userlogs and userlogs are generated afterwards).
+    if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
+      try {
+        LOG.info("FileUtil#symlink: On Java6, copying file instead "
+            + linkname + " -> " + target);
+        org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
+      } catch (IOException ex) {
+        LOG.warn("FileUtil#symlink failed to copy the file with error: "
+            + ex.getMessage());
+        // Exit with non-zero exit code
+        return 1;
+      }
+      return 0;
+    }
+
+    String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
+        linkFile.getPath());
     ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
     try {
       shExec.execute();
@@ -669,6 +763,25 @@ public class FileUtil {
   }
 
   /**
+   * Set the ownership on a file / directory. User name and group name
+   * cannot both be null.
+   * @param file the file to change
+   * @param username the new user owner name
+   * @param groupname the new group owner name
+   * @throws IOException
+   */
+  public static void setOwner(File file, String username,
+      String groupname) throws IOException {
+    if (username == null && groupname == null) {
+      throw new IOException("username == null && groupname == null");
+    }
+    String arg = (username == null ? "" : username)
+        + (groupname == null ? "" : ":" + groupname);
+    String [] cmd = Shell.getSetOwnerCommand(arg);
+    execCommand(file, cmd);
+  }
+
+  /**
    * Set permissions to the required value. Uses the java primitives instead
    * of forking if group == other.
    * @param f the file to change

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Sat Dec 15 16:29:47 2012
@@ -494,9 +494,9 @@ public class RawLocalFileSystem extends 
       return !super.getOwner().isEmpty(); 
     }
     
-    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
+    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { 
       super(f.length(), f.isDirectory(), 1, defaultBlockSize,
-            f.lastModified(), fs.makeQualified(new Path(f.getPath())));
+          f.lastModified(), new Path(f.getPath()).makeQualified(fs));
     }
     
     @Override
@@ -527,9 +527,10 @@ public class RawLocalFileSystem extends 
     private void loadPermissionInfo() {
       IOException e = null;
       try {
-        StringTokenizer t = new StringTokenizer(
-            FileUtil.execCommand(new File(getPath().toUri()), 
-                                          Shell.getGetPermissionCommand()));
+        String output = FileUtil.execCommand(new File(getPath().toUri()), 
+            Shell.getGetPermissionCommand());
+        StringTokenizer t =
+            new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
         //expected format
         //-rw-------    1 username groupname ...
         String permission = t.nextToken();
@@ -549,7 +550,6 @@ public class RawLocalFileSystem extends 
         }
         setOwner(owner);
 
-        // FIXME: Group names could have spaces on Windows
         setGroup(t.nextToken());
       } catch (Shell.ExitCodeException ioe) {
         if (ioe.getExitCode() != 1) {
@@ -585,17 +585,7 @@ public class RawLocalFileSystem extends 
   @Override
   public void setOwner(Path p, String username, String groupname)
     throws IOException {
-    if (username == null && groupname == null) {
-      throw new IOException("username == null && groupname == null");
-    }
-
-    if (username == null) {
-      FileUtil.execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
-    } else {
-      //OWNER[:[GROUP]]
-      String s = username + (groupname == null? "": ":" + groupname);
-      FileUtil.execCommand(pathToFile(p), Shell.getSetOwnerCommand(s));
-    }
+    FileUtil.setOwner(pathToFile(p), username, groupname);
   }
 
   /**

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java Sat Dec 15 16:29:47 2012
@@ -356,10 +356,6 @@ public class NativeIO {
     /** Windows only methods used for getOwner() implementation */
     private static native String getOwner(FileDescriptor fd) throws IOException;
 
-    /** Windows only method used for getting the file length */
-    public static native long getLengthFollowSymlink(
-        String path) throws IOException;
-
     static {
       if (NativeCodeLoader.isNativeCodeLoaded()) {
         try {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java Sat Dec 15 16:29:47 2012
@@ -86,7 +86,8 @@ public class ShellBasedUnixGroupsMapping
       LOG.warn("got exception trying to get groups for user " + user, e);
     }
     
-    StringTokenizer tokenizer = new StringTokenizer(result);
+    StringTokenizer tokenizer =
+        new StringTokenizer(result, Shell.TOKEN_SEPARATOR_REGEX);
     List<String> groups = new LinkedList<String>();
     while (tokenizer.hasMoreTokens()) {
       groups.add(tokenizer.nextToken());

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java Sat Dec 15 16:29:47 2012
@@ -45,9 +45,12 @@ abstract public class Shell {
   
   public static final Log LOG = LogFactory.getLog(Shell.class);
   
-  /** a Windows utility to emulate Unix commands */
-  public static final String WINUTILS = System.getenv("HADOOP_HOME")
-                                        + "\\bin\\winutils";
+  private static boolean IS_JAVA7_OR_ABOVE =
+      System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0;
+
+  public static boolean isJava7OrAbove() {
+    return IS_JAVA7_OR_ABOVE;
+  }
 
   /** a Unix command to get the current user's name */
   public final static String USER_NAME_COMMAND = "whoami";
@@ -64,7 +67,7 @@ abstract public class Shell {
   /** a Unix command to get a given user's groups list */
   public static String[] getGroupsForUserCommand(final String user) {
     //'groups username' command return is non-consistent across different unixes
-    return (WINDOWS)? new String[] { WINUTILS, "groups", user}
+    return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user + "\""}
                     : new String [] {"bash", "-c", "id -Gn " + user};
   }
 
@@ -77,7 +80,7 @@ abstract public class Shell {
 
   /** Return a command to get permission information. */
   public static String[] getGetPermissionCommand() {
-    return (WINDOWS) ? new String[] { WINUTILS, "ls" }
+    return (WINDOWS) ? new String[] { WINUTILS, "ls", "-F" }
                      : new String[] { "/bin/ls", "-ld" };
   }
 
@@ -110,10 +113,10 @@ abstract public class Shell {
 
   /** Return a command to set owner */
   public static String[] getSetOwnerCommand(String owner) {
-    return (WINDOWS) ? new String[] { WINUTILS, "chown", owner }
+    return (WINDOWS) ? new String[] { WINUTILS, "chown", "\"" + owner + "\"" }
                      : new String[] { "chown", owner };
   }
-
+  
   /** Return a command to create symbolic links */
   public static String[] getSymlinkCommand(String target, String link) {
     return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
@@ -163,6 +166,84 @@ abstract public class Shell {
   /** If or not script timed out*/
   private AtomicBoolean timedOut;
 
+
+  /** Centralized logic to discover and validate the sanity of the Hadoop 
+   *  home directory. Returns either NULL or a directory that exists and 
+   *  was specified via either -Dhadoop.home.dir or the HADOOP_HOME ENV 
+   *  variable.  This does a lot of work so it should only be called 
+   *  privately for initialization once per process.
+   **/
+  private static String checkHadoopHome() {
+
+    // first check the Dflag hadoop.home.dir with JVM scope
+    String home = System.getProperty("hadoop.home.dir");
+
+    // fall back to the system/user-global env variable
+    if (home == null) {
+      home = System.getenv("HADOOP_HOME");
+    }
+
+    try {
+       // couldn't find either setting for hadoop's home directory
+       if (home == null) {
+         throw new IOException("HADOOP_HOME or hadoop.home.dir are not set.");
+       }
+
+       if (home.startsWith("\"") && home.endsWith("\"")) {
+         home = home.substring(1, home.length()-1);
+       }
+
+       // check that the home setting is actually a directory that exists
+       File homedir = new File(home);
+       if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) {
+         throw new IOException("Hadoop home directory " + homedir
+           + " does not exist, is not a directory, or is not an absolute path.");
+       }
+
+       home = homedir.getCanonicalPath();
+
+    } catch (IOException ioe) {
+       LOG.error("Failed to detect a valid hadoop home directory", ioe);
+       home = null;
+    }
+    
+    return home;
+  }
+  private static String HADOOP_HOME_DIR = checkHadoopHome();
+
+  // Public getter, throws an exception if HADOOP_HOME failed validation
+  // checks and is being referenced downstream.
+  public static final String getHadoopHome() throws IOException {
+    if (HADOOP_HOME_DIR == null) {
+      throw new IOException("Misconfigured HADOOP_HOME cannot be referenced.");
+    }
+
+    return HADOOP_HOME_DIR;
+  }
+
+  /** fully qualify the path to a binary that should be in a known hadoop 
+   *  bin location. This is primarily useful for disambiguating call-outs 
+   *  to executable sub-components of Hadoop to avoid clashes with other 
+   *  executables that may be in the path.  Caveat:  this call doesn't 
+   *  just format the path to the bin directory.  It also checks for file 
+   *  existence of the composed path. The output of this call should be 
+   *  cached by callers.
+   * */
+  public static final String getQualifiedBinPath(String executable) 
+  throws IOException {
+    // construct hadoop bin path to the specified executable
+    String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" 
+      + File.separator + executable;
+
+    File exeFile = new File(fullExeName);
+    if (!exeFile.exists()) {
+      throw new IOException("Could not locate executable " + fullExeName
+        + " in the Hadoop binaries.");
+    }
+
+    return exeFile.getCanonicalPath();
+  }
+
   /** Set to true on Windows platforms */
   public static final boolean WINDOWS /* borrowed from Path.WINDOWS */
                 = System.getProperty("os.name").startsWith("Windows");
@@ -170,6 +251,28 @@ abstract public class Shell {
   public static final boolean LINUX
                 = System.getProperty("os.name").startsWith("Linux");
   
+  /** a Windows utility to emulate Unix commands */
+  public static final String WINUTILS = getWinUtilsPath();
+
+  public static final String getWinUtilsPath() {
+    String winUtilsPath = null;
+
+    try {
+      if (WINDOWS) {
+        winUtilsPath = getQualifiedBinPath("winutils.exe");
+      }
+    } catch (IOException ioe) {
+       LOG.error("Failed to locate the winutils binary in the hadoop binary path",
+         ioe);
+    }
+
+    return winUtilsPath;
+  }
+
+  /** Token separator regex used to parse Shell tool outputs */
+  public static final String TOKEN_SEPARATOR_REGEX
+                = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
+
   public static final boolean isSetsidAvailable = isSetsidSupported();
   private static boolean isSetsidSupported() {
     if (WINDOWS) {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c Sat Dec 15 16:29:47 2012
@@ -811,46 +811,6 @@ cleanup:
 #endif
 }
 
-/*
- * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
- * Method:    getLengthFollowSymlink
- * Signature: (Ljava/lang/String;)J
- *
- * The "00024" in the function name is an artifact of how JNI encodes
- * special characters. U+0024 is '$'.
- */
-JNIEXPORT jlong JNICALL
-Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_getLengthFollowSymlink
-  (JNIEnv *env, jclass clazz, jstring j_path)
-{
-#ifdef UNIX
-  THROW(env, "java/io/IOException",
-    "The function getLengthFollowSymlink(String) is not supported on Unix");
-  return 0;
-#endif
-
-#ifdef WINDOWS
-  DWORD dwRtnCode = ERROR_SUCCESS;
-  BY_HANDLE_FILE_INFORMATION fileInfo = { 0 };
-  LARGE_INTEGER fileSize = { 0 };
-
-  const wchar_t *path = (const wchar_t*) (*env)->GetStringChars(env, j_path, NULL);
-  if (path == NULL) return 0; // JVM throws Exception for us
-
-  dwRtnCode = GetFileInformationByName(path, TRUE, &fileInfo);
-  if (dwRtnCode != ERROR_SUCCESS) {
-    throw_ioe(env, dwRtnCode);
-  }
-
-  (*env)->ReleaseStringChars(env, j_path, path);
-
-  fileSize.HighPart = fileInfo.nFileSizeHigh;
-  fileSize.LowPart = fileInfo.nFileSizeLow;
-
-  return (jlong)(fileSize.QuadPart);
-#endif
-}
-
 /**
  * vim: sw=2: ts=2: et:
  */

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c Sat Dec 15 16:29:47 2012
@@ -263,6 +263,7 @@ static BOOL ChangeFileModeRecursively(__
   do
   {
     LPWSTR filename = NULL;
+    LPWSTR longFilename = NULL;
     size_t filenameSize = 0;
 
     if (wcscmp(ffd.cFileName, L".") == 0 ||
@@ -285,13 +286,25 @@ static BOOL ChangeFileModeRecursively(__
       goto ChangeFileModeRecursivelyEnd;
     }
      
-    if(!ChangeFileModeRecursively(filename, mode, actions))
+    // The child fileanme is not prepended with long path prefix.
+    // Convert the filename to long path format.
+    //
+    dwRtnCode = ConvertToLongPath(filename, &longFilename);
+    LocalFree(filename);
+    if (dwRtnCode != ERROR_SUCCESS)
     {
-      LocalFree(filename);
+      ReportErrorCode(L"ConvertToLongPath", dwRtnCode);
+      LocalFree(longFilename);
       goto ChangeFileModeRecursivelyEnd;
     }
 
-    LocalFree(filename);
+    if(!ChangeFileModeRecursively(longFilename, mode, actions))
+    {
+      LocalFree(longFilename);
+      goto ChangeFileModeRecursivelyEnd;
+    }
+
+    LocalFree(longFilename);
 
   } while (FindNextFileW(hFind, &ffd));
 
@@ -877,4 +890,4 @@ Change the mode of the FILE to MODE.\n\
 \n\
 Each MODE is of the form '[ugoa]*([-+=]([rwxX]*|[ugo]))+'.\n",
 program, program);
-}
\ No newline at end of file
+}

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chown.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chown.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chown.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/chown.c Sat Dec 15 16:29:47 2012
@@ -18,266 +18,92 @@
 #include "winutils.h"
 
 //----------------------------------------------------------------------------
-// Function: GetNewAclSize
+// Function: ChangeFileOwnerBySid
 //
 // Description:
-//	Compute the extra size of the new ACL if we replace the old owner Sid with
-//  the new owner Sid.
+//  Change a file or directory ownership by giving new owner and group SIDs
 //
 // Returns:
-//	The extra size needed for the new ACL compared with the ACL passed in. If
-//  the value is negative, it means the size of the new ACL could be reduced.
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
 //
 // Notes:
+//  This function is long path safe, i.e. the path will be converted to long
+//  path format if not already converted. So the caller does not need to do
+//  the converstion before calling the method.
 //
-static BOOL GetNewAclSizeDelta(__in PACL pDACL,
-  __in PSID pOldOwnerSid, __in PSID pNewOwnerSid, __out PLONG pDelta)
+static DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
+  __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
 {
-  PVOID pAce = NULL;
-  DWORD i;
-  PSID aceSid = NULL;
-  ACE_HEADER *aceHeader = NULL;
-  PACCESS_ALLOWED_ACE accessAllowedAce = NULL;
-  PACCESS_DENIED_ACE accessDenieddAce = NULL;
-
-  assert(pDACL != NULL && pNewOwnerSid != NULL &&
-    pOldOwnerSid != NULL && pDelta != NULL);
-
-  *pDelta = 0;
-  for (i = 0; i < pDACL->AceCount; i++)
-  {
-    if (!GetAce(pDACL, i, &pAce))
-    {
-      ReportErrorCode(L"GetAce", GetLastError());
-      return FALSE;
-    }
-
-    aceHeader = (ACE_HEADER *) pAce;
-    if (aceHeader->AceType == ACCESS_ALLOWED_ACE_TYPE)
-    {
-      accessAllowedAce = (PACCESS_ALLOWED_ACE) pAce;
-      aceSid = (PSID) &accessAllowedAce->SidStart;
-    }
-    else if (aceHeader->AceType == ACCESS_DENIED_ACE_TYPE)
-    {
-      accessDenieddAce = (PACCESS_DENIED_ACE) pAce;
-      aceSid = (PSID) &accessDenieddAce->SidStart;
-    }
-    else
-    {
-      continue;
-    }
-
-    if (EqualSid(pOldOwnerSid, aceSid))
-    {
-      *pDelta += GetLengthSid(pNewOwnerSid) - GetLengthSid(pOldOwnerSid);
-    }
-  }
-
-  return TRUE;
-}
-
-//----------------------------------------------------------------------------
-// Function: AddNewAce
-//
-// Description:
-//	Add an Ace of new owner to the new ACL
-//
-// Returns:
-//	TRUE: on success
-//
-// Notes:
-//  The Ace type should be either ACCESS_ALLOWED_ACE or ACCESS_DENIED_ACE
-//
-static BOOL AddNewAce(PACL pNewDACL, PVOID pOldAce,
-  PSID pOwnerSid, PSID pUserSid)
-{
-  PVOID pNewAce = NULL;
-  DWORD newAceSize = 0;
-
-  assert(pNewDACL != NULL && pOldAce != NULL &&
-    pOwnerSid != NULL && pUserSid != NULL);
-  assert(((PACE_HEADER)pOldAce)->AceType == ACCESS_ALLOWED_ACE_TYPE ||
-    ((PACE_HEADER)pOldAce)->AceType == ACCESS_DENIED_ACE_TYPE);
+  LPWSTR longPathName = NULL;
+  INT oldMode = 0;
 
-  newAceSize =  ((PACE_HEADER)pOldAce)->AceSize +
-    GetLengthSid(pUserSid) - GetLengthSid(pOwnerSid);
-  pNewAce = LocalAlloc(LPTR, newAceSize);
-  if (pNewAce == NULL)
-  {
-    ReportErrorCode(L"LocalAlloc", GetLastError());
-    return FALSE;
-  }
+  SECURITY_INFORMATION securityInformation = 0;
 
-  ((PACE_HEADER)pNewAce)->AceType = ((PACE_HEADER) pOldAce)->AceType;
-  ((PACE_HEADER)pNewAce)->AceFlags = ((PACE_HEADER) pOldAce)->AceFlags;
-  ((PACE_HEADER)pNewAce)->AceSize = (WORD) newAceSize;
+  DWORD dwRtnCode = ERROR_SUCCESS;
 
-  if (((PACE_HEADER)pOldAce)->AceType == ACCESS_ALLOWED_ACE_TYPE)
-  {
-    ((PACCESS_ALLOWED_ACE)pNewAce)->Mask = ((PACCESS_ALLOWED_ACE)pOldAce)->Mask;
-    if (!CopySid(GetLengthSid(pUserSid),
-      &((PACCESS_ALLOWED_ACE) pNewAce)->SidStart, pUserSid))
-    {
-      ReportErrorCode(L"CopySid", GetLastError());
-      LocalFree(pNewAce);
-      return FALSE;
-    }
-  }
-  else
-  {
-    ((PACCESS_DENIED_ACE)pNewAce)->Mask = ((PACCESS_DENIED_ACE)pOldAce)->Mask;
-    if (!CopySid(GetLengthSid(pUserSid),
-      &((PACCESS_DENIED_ACE) pNewAce)->SidStart, pUserSid))
-    {
-      ReportErrorCode(L"CopySid", GetLastError());
-      LocalFree(pNewAce);
-      return FALSE;
-    }
-  }
-
-  if (!AddAce(pNewDACL, ACL_REVISION, MAXDWORD,
-    pNewAce, ((PACE_HEADER)pNewAce)->AceSize))
+  // Convert the path the the long path
+  //
+  dwRtnCode = ConvertToLongPath(path, &longPathName);
+  if (dwRtnCode != ERROR_SUCCESS)
   {
-    ReportErrorCode(L"AddAce", GetLastError());
-    LocalFree(pNewAce);
-    return FALSE;
+    goto ChangeFileOwnerByNameEnd;
   }
 
-  LocalFree(pNewAce);
-  return TRUE;
-}
-
-//----------------------------------------------------------------------------
-// Function: CreateDaclForNewOwner
-//
-// Description:
-//	Create a new DACL for the new owner
-//
-// Returns:
-//	TRUE: on success
-//
-// Notes:
-//  Caller needs to destroy the memory of the new DACL by calling LocalFree()
-//
-static BOOL CreateDaclForNewOwner(
-  __in PACL pDACL,
-  __in_opt PSID pOldOwnerSid,
-  __in_opt PSID pNewOwnerSid,
-  __in_opt PSID pOldGroupSid,
-  __in_opt PSID pNewGroupSid,
-  __out PACL *ppNewDACL)
-{
-  PSID aceSid = NULL;
-  PACE_HEADER aceHeader = NULL;
-  PACCESS_ALLOWED_ACE accessAllowedAce = NULL;
-  PACCESS_DENIED_ACE accessDenieddAce = NULL;
-  PVOID pAce = NULL;
-  ACL_SIZE_INFORMATION aclSizeInfo;
-  LONG delta = 0;
-  DWORD dwNewAclSize = 0;
-  DWORD dwRtnCode = 0;
-  DWORD i;
-
-  assert(pDACL != NULL && ppNewDACL != NULL);
-  assert(pOldOwnerSid != NULL && pOldGroupSid != NULL);
-  assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
-
-  if (!GetAclInformation(pDACL, (LPVOID)&aclSizeInfo,
-    sizeof(ACL_SIZE_INFORMATION), AclSizeInformation))
+  // Get a pointer to the existing owner information and DACL
+  //
+  dwRtnCode = FindFileOwnerAndPermission(longPathName, NULL, NULL, &oldMode);
+  if (dwRtnCode != ERROR_SUCCESS)
   {
-    ReportErrorCode(L"GetAclInformation", GetLastError());
-    return FALSE;
+    goto ChangeFileOwnerByNameEnd;
   }
 
-  dwNewAclSize = aclSizeInfo.AclBytesInUse + aclSizeInfo.AclBytesFree;
-
-  delta = 0;
-  if (pNewOwnerSid != NULL &&
-    !GetNewAclSizeDelta(pDACL, pOldOwnerSid, pNewOwnerSid, &delta))
+  // We need SeTakeOwnershipPrivilege to set the owner if the caller does not
+  // have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
+  // SID is not contained in the caller's token, and have the SE_GROUP_OWNER
+  // permission enabled.
+  //
+  if (!EnablePrivilege(L"SeTakeOwnershipPrivilege"))
   {
-    return FALSE;
+    fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
   }
-  dwNewAclSize += delta;
-
-  delta = 0;
-  if (pNewGroupSid != NULL &&
-    !GetNewAclSizeDelta(pDACL, pOldGroupSid, pNewGroupSid, &delta))
+  if (!EnablePrivilege(L"SeRestorePrivilege"))
   {
-    return FALSE;
+    fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
   }
-  dwNewAclSize += delta;
 
-  *ppNewDACL = (PACL)LocalAlloc(LPTR, dwNewAclSize);
-  if (*ppNewDACL == NULL)
-  {
-    ReportErrorCode(L"LocalAlloc", GetLastError());
-    return FALSE;
-  }
+  assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
 
-  if (!InitializeAcl(*ppNewDACL, dwNewAclSize, ACL_REVISION))
+  // Set the owners of the file.
+  //
+  if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
+  if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
+  dwRtnCode = SetNamedSecurityInfoW(
+    longPathName,
+    SE_FILE_OBJECT,
+    securityInformation,
+    pNewOwnerSid,
+    pNewGroupSid,
+    NULL,
+    NULL);
+  if (dwRtnCode != ERROR_SUCCESS)
   {
-    ReportErrorCode(L"InitializeAcl", GetLastError());
-    return FALSE;
+    goto ChangeFileOwnerByNameEnd;
   }
 
-  // Go through the DACL to change permissions
+  // Set the permission on the file for the new owner.
   //
-  for (i = 0; i < pDACL->AceCount; i++)
+  dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
+  if (dwRtnCode != ERROR_SUCCESS)
   {
-    if (!GetAce(pDACL, i, &pAce))
-    {
-      ReportErrorCode(L"GetAce", GetLastError());
-      return FALSE;
-    }
-
-    aceHeader = (PACE_HEADER) pAce;
-    aceSid = NULL;
-    if (aceHeader->AceType == ACCESS_ALLOWED_ACE_TYPE)
-    {
-      accessAllowedAce = (PACCESS_ALLOWED_ACE) pAce;
-      aceSid = (PSID) &accessAllowedAce->SidStart;
-    }
-    else if (aceHeader->AceType == ACCESS_DENIED_ACE_TYPE)
-    {
-      accessDenieddAce = (PACCESS_DENIED_ACE) pAce;
-      aceSid = (PSID) &accessDenieddAce->SidStart;
-    }
-
-    if (aceSid != NULL)
-    {
-      if (pNewOwnerSid != NULL && EqualSid(pOldOwnerSid, aceSid))
-      {
-        if (!AddNewAce(*ppNewDACL, pAce, pOldOwnerSid, pNewOwnerSid))
-          return FALSE;
-        else
-          continue;
-      }
-      else if (pNewGroupSid != NULL && EqualSid(pOldGroupSid, aceSid))
-      {
-        if (!AddNewAce(*ppNewDACL, pAce, pOldGroupSid, pNewGroupSid))
-          return FALSE;
-        else
-          continue;
-      }
-    }
-
-    // At this point, either:
-    // 1. The Ace is not of type ACCESS_ALLOWED_ACE or ACCESS_DENIED_ACE;
-    // 2. The Ace does not belong to the owner
-    // For both cases, we just add the oringal Ace to the new ACL.
-    //
-    if (!AddAce(*ppNewDACL, ACL_REVISION, MAXDWORD, pAce, aceHeader->AceSize))
-    {
-      ReportErrorCode(L"AddAce", dwRtnCode);
-      return FALSE;
-    }
+    goto ChangeFileOwnerByNameEnd;
   }
 
-  return TRUE;
+ChangeFileOwnerByNameEnd:
+  LocalFree(longPathName);
+  return dwRtnCode;
 }
 
-
 //----------------------------------------------------------------------------
 // Function: Chown
 //
@@ -293,7 +119,6 @@ static BOOL CreateDaclForNewOwner(
 int Chown(int argc, wchar_t *argv[])
 {
   LPWSTR pathName = NULL;
-  LPWSTR longPathName = NULL;
 
   LPWSTR ownerInfo = NULL;
 
@@ -308,16 +133,6 @@ int Chown(int argc, wchar_t *argv[])
   PSID pNewOwnerSid = NULL;
   PSID pNewGroupSid = NULL;
 
-  PACL pDACL = NULL;
-  PACL pNewDACL = NULL;
-
-  PSID pOldOwnerSid = NULL;
-  PSID pOldGroupSid = NULL;
-
-  PSECURITY_DESCRIPTOR pSD = NULL;
-
-  SECURITY_INFORMATION securityInformation;
-
   DWORD dwRtnCode = 0;
 
   int ret = EXIT_FAILURE;
@@ -353,7 +168,7 @@ int Chown(int argc, wchar_t *argv[])
         goto ChownEnd;
     }
 
-    if (colonPos + 1 != NULL)
+    if (*(colonPos + 1) != 0)
     {
       // Length includes NULL terminator
       groupNameLen = wcslen(ownerInfo) - (colonPos - ownerInfo) + 1;
@@ -382,10 +197,16 @@ int Chown(int argc, wchar_t *argv[])
       goto ChownEnd;
   }
 
-  if ((userName == NULL || wcslen(userName) == 0) &&
-    (groupName == NULL || wcslen(groupName) == 0))
+  // Not allow zero length user name or group name in the parsing results.
+  //
+  assert(userName == NULL || wcslen(userName) > 0);
+  assert(groupName == NULL || wcslen(groupName) > 0);
+
+  // Nothing to change if both names are empty
+  //
+  if ((userName == NULL) && (groupName == NULL))
   {
-    fwprintf(stderr, L"User name and group name cannot both be empty.");
+    ret = EXIT_SUCCESS;
     goto ChownEnd;
   }
 
@@ -417,61 +238,10 @@ int Chown(int argc, wchar_t *argv[])
     goto ChownEnd;
   }
 
-  // Convert the path the the long path
-  //
-  dwRtnCode = ConvertToLongPath(pathName, &longPathName);
-  if (dwRtnCode != ERROR_SUCCESS)
-  {
-    ReportErrorCode(L"ConvertToLongPath", dwRtnCode);
-    goto ChownEnd;
-  }
-
-  // Get a pointer to the existing owner information and DACL
-  //
-  dwRtnCode = GetNamedSecurityInfoW(
-    longPathName,
-    SE_FILE_OBJECT, 
-    OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION |
-    DACL_SECURITY_INFORMATION,
-    &pOldOwnerSid,
-    &pOldGroupSid,
-    &pDACL,
-    NULL,
-    &pSD);
+  dwRtnCode = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
   if (dwRtnCode != ERROR_SUCCESS)
   {
-    ReportErrorCode(L"GetNamedSecurityInfo", dwRtnCode);
-    goto ChownEnd;
-  }
-
-  // Create the new DACL
-  //
-  if (!CreateDaclForNewOwner(pDACL,
-    pOldOwnerSid, pNewOwnerSid,
-    pOldGroupSid, pNewGroupSid,
-    &pNewDACL))
-  {
-    goto ChownEnd;
-  }
-
-  // Set the owner and DACLs in the object's security descriptor. Use
-  // PROTECTED_DACL_SECURITY_INFORMATION flag to remove permission inheritance
-  //
-  securityInformation =
-    DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION;
-  if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
-  if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
-  dwRtnCode = SetNamedSecurityInfoW(
-    longPathName,
-    SE_FILE_OBJECT,
-    securityInformation,
-    pNewOwnerSid,
-    pNewGroupSid,
-    pNewDACL,
-    NULL);
-  if (dwRtnCode != ERROR_SUCCESS)
-  {
-    ReportErrorCode(L"SetNamedSecurityInfo", dwRtnCode);
+    ReportErrorCode(L"ChangeFileOwnerBySid", dwRtnCode);
     goto ChownEnd;
   }
 
@@ -482,9 +252,6 @@ ChownEnd:
   LocalFree(groupName);
   LocalFree(pNewOwnerSid);
   LocalFree(pNewGroupSid);
-  LocalFree(pNewDACL);
-  LocalFree(pSD);
-  LocalFree(longPathName);
 
   return ret;
 }
@@ -493,6 +260,11 @@ void ChownUsage(LPCWSTR program)
 {
   fwprintf(stdout, L"\
 Usage: %s [OWNER][:[GROUP]] [FILE]\n\
-Change the owner and/or group of the FILE to OWNER and/or GROUP.\n",
+Change the owner and/or group of the FILE to OWNER and/or GROUP.\n\
+\n\
+Note:\n\
+On Linux, if a colon but no group name follows the user name, the group of\n\
+the files is changed to that user\'s login group. Windows has no concept of\n\
+a user's login group. So we do not change the group owner in this case.\n",
 program);
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/groups.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/groups.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/groups.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/groups.c Sat Dec 15 16:29:47 2012
@@ -28,8 +28,13 @@
 // Notes:
 //   This function could fail on first pass when we fail to find groups for
 //   domain account; so we do not report Windows API errors in this function.
+//   If formatOutput is true, pipe character is used as separator for groups
+//   otherwise, space.
 //
-static BOOL PrintGroups(LPLOCALGROUP_USERS_INFO_0 groups, DWORD entries)
+static BOOL PrintGroups(
+  LPLOCALGROUP_USERS_INFO_0 groups,
+  DWORD entries,
+  BOOL formatOutput)
 {
   BOOL ret = TRUE;
   LPLOCALGROUP_USERS_INFO_0 pTmpBuf = groups;
@@ -45,7 +50,14 @@ static BOOL PrintGroups(LPLOCALGROUP_USE
 
     if (i != 0)
     {
-      wprintf(L" ");
+      if (formatOutput)
+      {
+        wprintf(L"|");
+      }
+      else
+      {
+        wprintf(L" ");
+      }
     }
     wprintf(L"%s", pTmpBuf->lgrui0_name);
 
@@ -59,6 +71,56 @@ static BOOL PrintGroups(LPLOCALGROUP_USE
 }
 
 //----------------------------------------------------------------------------
+// Function: ParseCommandLine
+//
+// Description:
+//   Parses the command line
+//
+// Returns:
+//   TRUE on the valid command line, FALSE otherwise
+//
+static BOOL ParseCommandLine(
+  int argc, wchar_t *argv[], wchar_t **user, BOOL *formatOutput)
+{
+  *formatOutput = FALSE;
+
+  assert(argv != NULL);
+  assert(user != NULL);
+
+  if (argc == 1)
+  {
+    // implicitly use the current user
+    *user = NULL;
+    return TRUE;
+  }
+  else if (argc == 2)
+  {
+    // check if the second argument is formating
+    if (wcscmp(argv[1], L"-F") == 0)
+    {
+      *user = NULL;
+      *formatOutput = TRUE;
+      return TRUE;
+    }
+    else
+    {
+      *user = argv[1];
+      return TRUE;
+    }
+  }
+  else if (argc == 3 && wcscmp(argv[1], L"-F") == 0)
+  {
+    // if 3 args, the second argument must be "-F"
+
+    *user = argv[2];
+    *formatOutput = TRUE;
+    return TRUE;
+  }
+
+  return FALSE;
+}
+
+//----------------------------------------------------------------------------
 // Function: Groups
 //
 // Description:
@@ -83,15 +145,18 @@ int Groups(int argc, wchar_t *argv[])
   DWORD dwRtnCode = ERROR_SUCCESS;
 
   int ret = EXIT_SUCCESS;
+  BOOL formatOutput = FALSE;
 
-  if (argc != 2 && argc != 1)
+  if (!ParseCommandLine(argc, argv, &input, &formatOutput))
   {
     fwprintf(stderr, L"Incorrect command line arguments.\n\n");
     GroupsUsage(argv[0]);
     return EXIT_FAILURE;
   }
 
-  if (argc == 1)
+  // if username was not specified on the command line, fallback to the
+  // current user
+  if (input == NULL)
   {
     GetUserNameW(currentUser, &cchCurrentUser);
     if (GetLastError() == ERROR_INSUFFICIENT_BUFFER)
@@ -120,10 +185,6 @@ int Groups(int argc, wchar_t *argv[])
       goto GroupsEnd;
     }
   }
-  else
-  {
-    input = argv[1];
-  }
 
   if ((dwRtnCode = GetLocalGroupsForUser(input, &groups, &entries))
     != ERROR_SUCCESS)
@@ -133,7 +194,7 @@ int Groups(int argc, wchar_t *argv[])
     goto GroupsEnd;
   }
 
-  if (!PrintGroups(groups, entries))
+  if (!PrintGroups(groups, entries, formatOutput))
   {
     ret = EXIT_FAILURE;
   }
@@ -147,8 +208,10 @@ GroupsEnd:
 void GroupsUsage(LPCWSTR program)
 {
   fwprintf(stdout, L"\
-Usage: %s [USERNAME]\n\
-Print group information of the specified USERNAME\
-(the current user by default).\n",
+Usage: %s [OPTIONS] [USERNAME]\n\
+Print group information of the specified USERNAME \
+(the current user by default).\n\
+\n\
+OPTIONS: -F format the output by separating tokens with a pipe\n",
 program);
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h Sat Dec 15 16:29:47 2012
@@ -137,4 +137,6 @@ DWORD JunctionPointCheck(__in LPCWSTR pa
 DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode);
 
 DWORD GetLocalGroupsForUser(__in LPCWSTR user,
-  __out LPLOCALGROUP_USERS_INFO_0 *groups, __out LPDWORD entries);
\ No newline at end of file
+  __out LPLOCALGROUP_USERS_INFO_0 *groups, __out LPDWORD entries);
+
+BOOL EnablePrivilege(__in LPCWSTR privilegeName);
\ No newline at end of file

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c Sat Dec 15 16:29:47 2012
@@ -144,7 +144,7 @@ static BOOL IsLongWindowsPath(__in PCWST
 static BOOL IsPrefixedAlready(__in PCWSTR path)
 {
   static const PCWSTR LongPathPrefix = L"\\\\?\\";
-  int Prefixlen = (int)wcslen(LongPathPrefix);
+  size_t Prefixlen = wcslen(LongPathPrefix);
   size_t i = 0;
 
   if (path == NULL || wcslen(path) < Prefixlen)
@@ -413,10 +413,13 @@ DWORD GetSidFromAcctNameW(LPCWSTR acctNa
   assert (acctName != NULL && ppSid != NULL);
 
   // Empty name is invalid. However, LookupAccountName() function will return a
-  // false Sid for an empty name instead failing.
+  // false Sid, i.e. Sid for 'BUILDIN', for an empty name instead failing. We
+  // report the error before calling LookupAccountName() function for this
+  // special case. The error code returned here is the same as the last error
+  // code set by LookupAccountName() function for an invalid name.
   //
   if (wcslen(acctName) == 0)
-    return FALSE;
+    return ERROR_NONE_MAPPED;
 
   // First pass to retrieve the buffer size.
   //
@@ -478,7 +481,7 @@ DWORD GetSidFromAcctNameW(LPCWSTR acctNa
 //	Compute the 3 bit Unix mask for the owner, group, or, others
 //
 // Returns:
-//	The 3 bit Unix mask in USHORT
+//	The 3 bit Unix mask in INT
 //
 // Notes:
 //
@@ -613,8 +616,11 @@ GetEffectiveRightsForSidEnd:
 //  Error code otherwise
 //
 // Notes:
-//  Caller needs to destroy the memeory of owner and group names by calling
-//  LocalFree() function.
+//  - Caller needs to destroy the memeory of owner and group names by calling
+//    LocalFree() function.
+//
+//  - If the user or group name does not exist, the user or group SID will be
+//    returned as the name.
 //
 DWORD FindFileOwnerAndPermission(
   __in LPCWSTR pathName,
@@ -830,7 +836,17 @@ static void GetWindowsAccessMask(INT uni
 //  Error code: otherwise
 //
 // Notes:
-//  none
+//  - Administrators and SYSTEM are always given full permission to the file,
+//    unless Administrators or SYSTEM itself is the file owner and the user
+//    explictly set the permission to something else. For example, file 'foo'
+//    belongs to Administrators, 'chmod 000' on the file will not directly
+//    assign Administrators full permission on the file.
+//  - Only full permission for Administrators and SYSTEM are inheritable.
+//  - CREATOR OWNER is always given full permission and the permission is
+//    inheritable, more specifically OBJECT_INHERIT_ACE, CONTAINER_INHERIT_ACE
+//    flags are set. The reason is to give the creator of child file full
+//    permission, i.e., the child file will have permission mode 700 for
+//    a user other than Administrator or SYSTEM.
 //
 static DWORD GetWindowsDACLs(__in INT unixMask,
   __in PSID pOwnerSid, __in PSID pGroupSid, __out PACL *ppNewDACL)
@@ -842,12 +858,22 @@ static DWORD GetWindowsDACLs(__in INT un
   DWORD winOtherAccessAllowMask;
 
   PSID pEveryoneSid = NULL;
+  DWORD cbEveryoneSidSize = SECURITY_MAX_SID_SIZE;
+
+  PSID pSystemSid = NULL;
+  DWORD cbSystemSidSize = SECURITY_MAX_SID_SIZE;
+  BOOL bAddSystemAcls = FALSE;
+
+  PSID pAdministratorsSid = NULL;
+  DWORD cbAdministratorsSidSize = SECURITY_MAX_SID_SIZE;
+  BOOL bAddAdministratorsAcls = FALSE;
+
+  PSID pCreatorOwnerSid = NULL;
+  DWORD cbCreatorOwnerSidSize = SECURITY_MAX_SID_SIZE;
 
   PACL pNewDACL = NULL;
   DWORD dwNewAclSize = 0;
 
-  SID_IDENTIFIER_AUTHORITY SIDAuthWorld = SECURITY_WORLD_SID_AUTHORITY;
-
   DWORD ret = ERROR_SUCCESS;
 
   GetWindowsAccessMask(unixMask,
@@ -857,12 +883,63 @@ static DWORD GetWindowsDACLs(__in INT un
 
   // Create a well-known SID for the Everyone group
   //
-  if(!AllocateAndInitializeSid(&SIDAuthWorld, 1,
-    SECURITY_WORLD_RID,
-    0, 0, 0, 0, 0, 0, 0,
-    &pEveryoneSid))
+  if ((pEveryoneSid = LocalAlloc(LPTR, cbEveryoneSidSize)) == NULL)
   {
-    return GetLastError();
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinWorldSid, NULL, pEveryoneSid, &cbEveryoneSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  // Create a well-known SID for the Administrators group
+  //
+  if ((pAdministratorsSid = LocalAlloc(LPTR, cbAdministratorsSidSize)) == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL,
+    pAdministratorsSid, &cbAdministratorsSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!EqualSid(pAdministratorsSid, pOwnerSid)
+    && !EqualSid(pAdministratorsSid, pGroupSid))
+    bAddAdministratorsAcls = TRUE;
+
+  // Create a well-known SID for the SYSTEM
+  //
+  if ((pSystemSid = LocalAlloc(LPTR, cbSystemSidSize)) == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinLocalSystemSid, NULL,
+    pSystemSid, &cbSystemSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!EqualSid(pSystemSid, pOwnerSid)
+    && !EqualSid(pSystemSid, pGroupSid))
+    bAddSystemAcls = TRUE;
+
+  // Create a well-known SID for the Creator Owner
+  //
+  if ((pCreatorOwnerSid = LocalAlloc(LPTR, cbCreatorOwnerSidSize)) == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinCreatorOwnerSid, NULL,
+    pCreatorOwnerSid, &cbCreatorOwnerSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
   }
 
   // Create the new DACL
@@ -880,6 +957,22 @@ static DWORD GetWindowsDACLs(__in INT un
     GetLengthSid(pGroupSid) - sizeof(DWORD);
   dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
     GetLengthSid(pEveryoneSid) - sizeof(DWORD);
+
+  if (bAddSystemAcls)
+  {
+    dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+      cbSystemSidSize - sizeof(DWORD);
+  }
+
+  if (bAddAdministratorsAcls)
+  {
+    dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+      cbAdministratorsSidSize - sizeof(DWORD);
+  }
+
+  dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+    cbCreatorOwnerSidSize - sizeof(DWORD);
+
   pNewDACL = (PACL)LocalAlloc(LPTR, dwNewAclSize);
   if (pNewDACL == NULL)
   {
@@ -892,33 +985,64 @@ static DWORD GetWindowsDACLs(__in INT un
     goto GetWindowsDACLsEnd;
   }
 
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
+    GENERIC_ALL, pCreatorOwnerSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  if (bAddSystemAcls &&
+    !AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
+    GENERIC_ALL, pSystemSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  if (bAddAdministratorsAcls &&
+    !AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
+    GENERIC_ALL, pAdministratorsSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
   if (winUserAccessDenyMask &&
-    !AddAccessDeniedAce(pNewDACL, ACL_REVISION,
+    !AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
     winUserAccessDenyMask, pOwnerSid))
   {
     ret = GetLastError();
     goto GetWindowsDACLsEnd;
   }
-  if (!AddAccessAllowedAce(pNewDACL, ACL_REVISION,
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
     winUserAccessAllowMask, pOwnerSid))
   {
     ret = GetLastError();
     goto GetWindowsDACLsEnd;
   }
   if (winGroupAccessDenyMask &&
-    !AddAccessDeniedAce(pNewDACL, ACL_REVISION,
+    !AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
     winGroupAccessDenyMask, pGroupSid))
   {
     ret = GetLastError();
     goto GetWindowsDACLsEnd;
   }
-  if (!AddAccessAllowedAce(pNewDACL, ACL_REVISION,
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
     winGroupAccessAllowMask, pGroupSid))
   {
     ret = GetLastError();
     goto GetWindowsDACLsEnd;
   }
-  if (!AddAccessAllowedAce(pNewDACL, ACL_REVISION,
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
     winOtherAccessAllowMask, pEveryoneSid))
   {
     ret = GetLastError();
@@ -928,7 +1052,10 @@ static DWORD GetWindowsDACLs(__in INT un
   *ppNewDACL = pNewDACL;
 
 GetWindowsDACLsEnd:
-  if (pEveryoneSid) FreeSid(pEveryoneSid);
+  LocalFree(pEveryoneSid);
+  LocalFree(pAdministratorsSid);
+  LocalFree(pSystemSid);
+  LocalFree(pCreatorOwnerSid);
   if (ret != ERROR_SUCCESS) LocalFree(pNewDACL);
   
   return ret;
@@ -952,7 +1079,6 @@ GetWindowsDACLsEnd:
 DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode)
 {
   LPWSTR longPathName = NULL;
-  PACL pOldDACL = NULL;
   PACL pNewDACL = NULL;
   PSID pOwnerSid = NULL;
   PSID pGroupSid = NULL;
@@ -984,11 +1110,10 @@ DWORD ChangeFileModeByMask(__in LPCWSTR 
   dwRtnCode = GetNamedSecurityInfoW(
     longPathName,
     SE_FILE_OBJECT, 
-    OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION |
-    DACL_SECURITY_INFORMATION,
+    OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION,
     &pOwnerSid,
     &pGroupSid,
-    &pOldDACL,
+    NULL,
     NULL,
     &pSD);
   if (ERROR_SUCCESS != dwRtnCode)
@@ -1221,12 +1346,15 @@ GetAccntNameFromSidEnd:
 //  Other error code on failure
 //
 // Notes:
-// - Because NetUserGetLocalGroups() only accepts full user name, we wil try
-//   to find full user name given the user input if NetUserGetLocalGroups()
-//   fails on first time with error code NERR_UserNotFound. It is not always
-//   to find full user name given only user name. For example, a computer named
-//   'win1' joint domain 'redmond' can have both 'win1\alex' and
-//   'redmond\alex'. Given only alex, we cannot tell which one is correct.
+// - NetUserGetLocalGroups() function only accepts full user name in the format
+//   [domain name]\[username]. The user input to this function can be only the
+//   username. In this case, NetUserGetLocalGroups() will fail on the first try,
+//   and we will try to find full user name using LookupAccountNameW() method,
+//   and call NetUserGetLocalGroups() function again with full user name.
+//   However, it is not always possible to find full user name given only user
+//   name. For example, a computer named 'win1' joined domain 'redmond' can have
+//   two different users, 'win1\alex' and 'redmond\alex'. Given only 'alex', we
+//   cannot tell which one is correct.
 //
 // - Caller needs to destroy the memory of groups by using the
 //   NetApiBufferFree() function
@@ -1307,6 +1435,51 @@ GetLocalGroupsForUserEnd:
 }
 
 //----------------------------------------------------------------------------
+// Function: EnablePrivilege
+//
+// Description:
+//	Check if the process has the given privilege. If yes, enable the privilege
+//  to the process's access token.
+//
+// Returns:
+//	TRUE: on success
+//
+// Notes:
+//
+BOOL EnablePrivilege(__in LPCWSTR privilegeName)
+{
+  HANDLE hToken = INVALID_HANDLE_VALUE;
+  TOKEN_PRIVILEGES tp = { 0 };
+  DWORD dwErrCode = ERROR_SUCCESS;
+
+  if (!OpenProcessToken(GetCurrentProcess(),
+    TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken))
+  {
+    ReportErrorCode(L"OpenProcessToken", GetLastError());
+    return FALSE;
+  }
+
+  tp.PrivilegeCount = 1;
+  if (!LookupPrivilegeValueW(NULL,
+    privilegeName, &(tp.Privileges[0].Luid)))
+  {
+    ReportErrorCode(L"LookupPrivilegeValue", GetLastError());
+    CloseHandle(hToken);
+    return FALSE;
+  }
+  tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+
+  // As stated on MSDN, we need to use GetLastError() to check if
+  // AdjustTokenPrivileges() adjusted all of the specified privileges.
+  //
+  AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL);
+  dwErrCode = GetLastError();
+  CloseHandle(hToken);
+
+  return dwErrCode == ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
 // Function: ReportErrorCode
 //
 // Description:

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/ls.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/ls.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/ls.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/ls.c Sat Dec 15 16:29:47 2012
@@ -76,6 +76,8 @@ static BOOL GetMaskString(INT accessMask
 //	None
 //
 // Notes:
+//  if useSeparator is false, separates the output tokens with a space
+//  character, otherwise, with a pipe character
 //
 static BOOL LsPrintLine(
   const INT mask,
@@ -84,7 +86,8 @@ static BOOL LsPrintLine(
   LPCWSTR groupName,
   const FILETIME *lpFileWritetime,
   const LARGE_INTEGER fileSize,
-  LPCWSTR path)
+  LPCWSTR path,
+  BOOL useSeparator)
 {
   // 'd' + 'rwx' for user, group, other
   static const size_t ck_ullMaskLen = 1 + 3 * 3;
@@ -117,10 +120,20 @@ static BOOL LsPrintLine(
     goto LsPrintLineEnd;
   }
 
-  fwprintf(stdout, L"%10s %d %s %s %lld %3s %2d %4d %s\n",
-    maskString, hardlinkCount, ownerName, groupName, fileSize.QuadPart,
-    MONTHS[stFileWriteTime.wMonth-1], stFileWriteTime.wDay,
-    stFileWriteTime.wYear, path);
+  if (useSeparator)
+  {
+    fwprintf(stdout, L"%10s|%d|%s|%s|%lld|%3s|%2d|%4d|%s\n",
+      maskString, hardlinkCount, ownerName, groupName, fileSize.QuadPart,
+      MONTHS[stFileWriteTime.wMonth-1], stFileWriteTime.wDay,
+      stFileWriteTime.wYear, path);
+  }
+  else
+  {
+    fwprintf(stdout, L"%10s %d %s %s %lld %3s %2d %4d %s\n",
+      maskString, hardlinkCount, ownerName, groupName, fileSize.QuadPart,
+      MONTHS[stFileWriteTime.wMonth-1], stFileWriteTime.wDay,
+      stFileWriteTime.wYear, path);
+  }
 
   ret = TRUE;
 
@@ -130,6 +143,88 @@ LsPrintLineEnd:
   return ret;
 }
 
+// List of command line options supported by "winutils ls"
+enum CmdLineOption
+{
+  CmdLineOptionFollowSymlink = 0x1,  // "-L"
+  CmdLineOptionSeparator = 0x2  // "-F"
+  // options should be powers of 2 (aka next is 0x4)
+};
+
+static wchar_t* CurrentDir = L".";
+
+//----------------------------------------------------------------------------
+// Function: ParseCommandLine
+//
+// Description:
+//   Parses the command line
+//
+// Returns:
+//   TRUE on the valid command line, FALSE otherwise
+//
+BOOL ParseCommandLine(
+  int argc, wchar_t *argv[], wchar_t** path, int *optionsMask)
+{
+  int MaxOptions = 2; // Should be equal to the number of elems in CmdLineOption
+  int i = 0;
+
+  assert(optionsMask != NULL);
+  assert(argv != NULL);
+  assert(path != NULL);
+
+  *optionsMask = 0;
+
+  if (argc == 1)
+  {
+    // no path specified, assume "."
+    *path = CurrentDir;
+    return TRUE;
+  }
+
+  if (argc == 2)
+  {
+    // only path specified, no other options
+    *path = argv[1];
+    return TRUE;
+  }
+
+  if (argc > 2 + MaxOptions)
+  {
+    // too many parameters
+    return FALSE;
+  }
+
+  for (i = 1; i < argc - 1; ++i)
+  {
+    if (wcscmp(argv[i], L"-L") == 0)
+    {
+      // Check if this option was already specified
+      BOOL alreadySet = *optionsMask & CmdLineOptionFollowSymlink;
+      if (alreadySet)
+        return FALSE;
+
+      *optionsMask |= CmdLineOptionFollowSymlink;
+    }
+    else if (wcscmp(argv[i], L"-F") == 0)
+    {
+      // Check if this option was already specified
+      BOOL alreadySet = *optionsMask & CmdLineOptionSeparator;
+      if (alreadySet)
+        return FALSE;
+
+      *optionsMask |= CmdLineOptionSeparator;
+    }
+    else
+    {
+      return FALSE;
+    }
+  }
+
+  *path = argv[argc - 1];
+
+  return TRUE;
+}
+
 //----------------------------------------------------------------------------
 // Function: Ls
 //
@@ -158,19 +253,16 @@ int Ls(int argc, wchar_t *argv[])
   BOOL isSymlink = FALSE;
 
   int ret = EXIT_FAILURE;
+  int optionsMask = 0;
 
-  if (argc > 2)
+  if (!ParseCommandLine(argc, argv, &pathName, &optionsMask))
   {
     fwprintf(stderr, L"Incorrect command line arguments.\n\n");
     LsUsage(argv[0]);
     return EXIT_FAILURE;
   }
 
-  if (argc == 2)
-    pathName = argv[1];
-
-  if (pathName == NULL || wcslen(pathName) == 0)
-    pathName = L".";
+  assert(pathName != NULL);
 
   if (wcsspn(pathName, L"/?|><:*\"") != 0)
   {
@@ -187,14 +279,15 @@ int Ls(int argc, wchar_t *argv[])
     goto LsEnd;
   }
 
-  dwErrorCode = GetFileInformationByName(longPathName, FALSE, &fileInformation);
+  dwErrorCode = GetFileInformationByName(
+    longPathName, optionsMask & CmdLineOptionFollowSymlink, &fileInformation);
   if (dwErrorCode != ERROR_SUCCESS)
   {
     ReportErrorCode(L"GetFileInformationByName", dwErrorCode);
     goto LsEnd;
   }
 
-  dwErrorCode = SymbolicLinkCheck(pathName, &isSymlink);
+  dwErrorCode = SymbolicLinkCheck(longPathName, &isSymlink);
   if (dwErrorCode != ERROR_SUCCESS)
   {
      ReportErrorCode(L"IsSymbolicLink", dwErrorCode);
@@ -224,7 +317,8 @@ int Ls(int argc, wchar_t *argv[])
     ownerName, groupName,
     &fileInformation.ftLastWriteTime,
     fileSize,
-    pathName))
+    pathName,
+    optionsMask & CmdLineOptionSeparator))
     goto LsEnd;
 
   ret = EXIT_SUCCESS;
@@ -240,10 +334,13 @@ LsEnd:
 void LsUsage(LPCWSTR program)
 {
   fwprintf(stdout, L"\
-Usage: %s [FILE]\n\
+Usage: %s [OPTIONS] [FILE]\n\
 List information about the FILE (the current directory by default).\n\
 Using long listing format and list directory entries instead of contents,\n\
 and do not dereference symbolic links.\n\
-Provide equivalent or similar function as 'ls -ld' on GNU/Linux.\n",
+Provides equivalent or similar function as 'ls -ld' on GNU/Linux.\n\
+\n\
+OPTIONS: -L dereference symbolic links\n\
+         -F format the output by separating tokens with a pipe\n",
 program);
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/main.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/main.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/main.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/main.c Sat Dec 15 16:29:47 2012
@@ -110,10 +110,9 @@ The available commands and their usages 
 
   fwprintf(stdout, L"%-15s%s\n\n", L"systeminfo", L"System information.");
   SystemInfoUsage();
+  fwprintf(stdout, L"\n\n");
 
   fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations.");
   TaskUsage();
   fwprintf(stdout, L"\n\n");
-
-  fwprintf(stdout, L"\n\n");
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c Sat Dec 15 16:29:47 2012
@@ -18,51 +18,6 @@
 #include "winutils.h"
 
 //----------------------------------------------------------------------------
-// Function: EnablePrivilege
-//
-// Description:
-//	Check if the process has the given privilege. If yes, enable the privilege
-//  to the process's access token.
-//
-// Returns:
-//	TRUE: on success
-//
-// Notes:
-//
-static BOOL EnablePrivilege(__in LPCWSTR privilegeName)
-{
-  HANDLE hToken;
-  TOKEN_PRIVILEGES tp;
-  DWORD dwErrCode;
-
-  if (!OpenProcessToken(GetCurrentProcess(),
-    TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken))
-  {
-    ReportErrorCode(L"OpenProcessToken", GetLastError());
-    return FALSE;
-  }
-
-  tp.PrivilegeCount = 1;
-  if (!LookupPrivilegeValueW(NULL,
-    privilegeName, &(tp.Privileges[0].Luid)))
-  {
-    ReportErrorCode(L"LookupPrivilegeValue", GetLastError());
-    CloseHandle(hToken);
-    return FALSE;
-  }
-  tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
-
-  // As stated on MSDN, we need to use GetLastError() to check if
-  // AdjustTokenPrivileges() adjusted all of the specified privileges.
-  //
-  AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL);
-  dwErrCode = GetLastError();
-  CloseHandle(hToken);
-
-  return dwErrCode == ERROR_SUCCESS;
-}
-
-//----------------------------------------------------------------------------
 // Function: Symlink
 //
 // Description:

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java Sat Dec 15 16:29:47 2012
@@ -509,7 +509,7 @@ public class TestFileUtil {
 
     //ensure that symlink length is correctly reported by Java
     Assert.assertEquals(data.length, file.length());
-    Assert.assertEquals(Shell.WINDOWS ? 0 : data.length, link.length());
+    Assert.assertEquals(data.length, link.length());
 
     //ensure that we can read from link.
     FileInputStream in = new FileInputStream(link);
@@ -520,4 +520,142 @@ public class TestFileUtil {
     in.close();
     Assert.assertEquals(data.length, len);
   }
+  
+  /**
+   * Test that rename on a symlink works as expected.
+   */
+  @Test
+  public void testSymlinkRenameTo() throws Exception {
+    Assert.assertFalse(del.exists());
+    del.mkdirs();
+
+    File file = new File(del, FILE);
+    file.createNewFile();
+    File link = new File(del, "_link");
+
+    // create the symlink
+    FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+    Assert.assertTrue(file.exists());
+    Assert.assertTrue(link.exists());
+
+    File link2 = new File(del, "_link2");
+
+    // Rename the symlink
+    Assert.assertTrue(link.renameTo(link2));
+
+    // Make sure the file still exists
+    // (NOTE: this would fail on Java6 on Windows if we didn't
+    // copy the file in FileUtil#symlink)
+    Assert.assertTrue(file.exists());
+
+    Assert.assertTrue(link2.exists());
+    Assert.assertFalse(link.exists());
+  }
+
+  /**
+   * Test that deletion of a symlink works as expected.
+   */
+  @Test
+  public void testSymlinkDelete() throws Exception {
+    Assert.assertFalse(del.exists());
+    del.mkdirs();
+
+    File file = new File(del, FILE);
+    file.createNewFile();
+    File link = new File(del, "_link");
+
+    // create the symlink
+    FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+    Assert.assertTrue(file.exists());
+    Assert.assertTrue(link.exists());
+
+    // make sure that deleting a symlink works properly
+    Assert.assertTrue(link.delete());
+    Assert.assertFalse(link.exists());
+    Assert.assertTrue(file.exists());
+  }
+
+  /**
+   * Test that length on a symlink works as expected.
+   */
+  @Test
+  public void testSymlinkLength() throws Exception {
+    Assert.assertFalse(del.exists());
+    del.mkdirs();
+
+    byte[] data = "testSymLinkData".getBytes();
+
+    File file = new File(del, FILE);
+    File link = new File(del, "_link");
+
+    // write some data to the file
+    FileOutputStream os = new FileOutputStream(file);
+    os.write(data);
+    os.close();
+
+    Assert.assertEquals(0, link.length());
+
+    // create the symlink
+    FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+    // ensure that File#length returns the target file and link size
+    Assert.assertEquals(data.length, file.length());
+    Assert.assertEquals(data.length, link.length());
+
+    file.delete();
+    Assert.assertFalse(file.exists());
+
+    if (Shell.WINDOWS && !Shell.isJava7OrAbove()) {
+      // On Java6 on Windows, we copied the file
+      Assert.assertEquals(data.length, link.length());
+    } else {
+      // Otherwise, the target file size is zero
+      Assert.assertEquals(0, link.length());
+    }
+
+    link.delete();
+    Assert.assertFalse(link.exists());
+  }
+
+  private void doUntarAndVerify(File tarFile, File untarDir) 
+                                 throws IOException {
+    if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) {
+      throw new IOException("Could not delete directory '" + untarDir + "'");
+    }
+    FileUtil.unTar(tarFile, untarDir);
+
+    String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name";
+    File testFile = new File(parentDir + Path.SEPARATOR + "version");
+    Assert.assertTrue(testFile.exists());
+    Assert.assertTrue(testFile.length() == 0);
+    String imageDir = parentDir + Path.SEPARATOR + "image";
+    testFile = new File(imageDir + Path.SEPARATOR + "fsimage");
+    Assert.assertTrue(testFile.exists());
+    Assert.assertTrue(testFile.length() == 157);
+    String currentDir = parentDir + Path.SEPARATOR + "current";
+    testFile = new File(currentDir + Path.SEPARATOR + "fsimage");
+    Assert.assertTrue(testFile.exists());
+    Assert.assertTrue(testFile.length() == 4331);
+    testFile = new File(currentDir + Path.SEPARATOR + "edits");
+    Assert.assertTrue(testFile.exists());
+    Assert.assertTrue(testFile.length() == 1033);
+    testFile = new File(currentDir + Path.SEPARATOR + "fstime");
+    Assert.assertTrue(testFile.exists());
+    Assert.assertTrue(testFile.length() == 8);
+  }
+
+  @Test
+  public void testUntar() throws IOException {
+    String tarGzFileName = System.getProperty("test.cache.data",
+        "build/test/cache") + "/test-untar.tgz";
+    String tarFileName = System.getProperty("test.cache.data",
+        "build/test/cache") + "/test-untar.tar";
+    String dataDir = System.getProperty("test.build.data", "build/test/data");
+    File untarDir = new File(dataDir, "untarDir");
+
+    doUntarAndVerify(new File(tarGzFileName), untarDir);
+    doUntarAndVerify(new File(tarFileName), untarDir);
+  }
 }

Added: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar?rev=1422280&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz?rev=1422280&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1422280&r1=1422279&r2=1422280&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java Sat Dec 15 16:29:47 2012
@@ -188,14 +188,15 @@ public class TestUserGroupInformation {
     }
     // get the groups
     pp = Runtime.getRuntime().exec(Shell.WINDOWS ?
-      Shell.WINUTILS + " groups" : "id -Gn");
+      Shell.WINUTILS + " groups -F" : "id -Gn");
     br = new BufferedReader(new InputStreamReader(pp.getInputStream()));
     String line = br.readLine();
 
     System.out.println(userName + ":" + line);
    
     Set<String> groups = new LinkedHashSet<String> ();    
-    for(String s: line.split("[\\s]")) {
+    String[] tokens = line.split(Shell.TOKEN_SEPARATOR_REGEX);
+    for(String s: tokens) {
       groups.add(s);
     }
     



Mime
View raw message