hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1485845 [1/4] - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/bin/ src/main/conf/ src/main/docs/src/documentation/content/xdocs/ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/ap...
Date Thu, 23 May 2013 20:41:57 GMT
Author: suresh
Date: Thu May 23 20:41:53 2013
New Revision: 1485845

URL: http://svn.apache.org/r1485845
Log:
HADOOP-8562. Merge r1453486 from trunk

Added:
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/native.sln
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/native.sln
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMappingWin.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMappingWin.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/
      - copied from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/chown.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/chown.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/groups.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/groups.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/hardlink.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/hardlink.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/include/
      - copied from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/include/
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/ls.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/ls.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/main.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/main.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/systeminfo.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/systeminfo.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/task.c
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/task.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
      - copied unchanged from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java
      - copied, changed from r1453486, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java
Modified:
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/rcc
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIOException.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/overview.html
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu May 23 20:41:53 2013
@@ -503,15 +503,15 @@ Release 2.0.3-alpha - 2013-02-06 
     HADOOP-9124. SortedMapWritable violates contract of Map interface for
     equals() and hashCode(). (Surenkumar Nihalani via tomwhite)
 
+    HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in
+    HarFileSystem. (Chris Nauroth via szetszwo)
+
     HADOOP-9252. In StringUtils, humanReadableInt(..) has a race condition and
     the synchronization of limitDecimalTo2(double) can be avoided.  (szetszwo)
 
     HADOOP-9260. Hadoop version may be not correct when starting name node or
     data node. (Chris Nauroth via jlowe)
 
-    HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in
-    HarFileSystem. (Chris Nauroth via szetszwo)
-
     HADOOP-9289. FsShell rm -f fails for non-matching globs. (Daryn Sharp via
     suresh)
 

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml Thu May 23 20:41:53 2013
@@ -415,6 +415,7 @@
         <configuration>
           <excludes>
             <exclude>CHANGES.txt</exclude>
+            <exclude>CHANGES.branch-trunk-win.txt</exclude>
             <exclude>.idea/**</exclude>
             <exclude>src/main/conf/*</exclude>
             <exclude>src/main/docs/**</exclude>
@@ -460,6 +461,28 @@
       <build>
         <plugins>
           <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-os</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireOS>
+                      <family>mac</family>
+                      <family>unix</family>
+                      <message>native build only supported on Mac or Unix</message>
+                    </requireOS>
+                  </rules>  
+                  <fail>true</fail>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>native-maven-plugin</artifactId>
             <executions>
@@ -531,5 +554,103 @@
         </plugins>
       </build>
     </profile>
+
+    <profile>
+      <id>native-win</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-os</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireOS>
+                      <family>windows</family>
+                      <message>native-win build only supported on Windows</message>
+                    </requireOS>
+                  </rules>  
+                  <fail>true</fail>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>native-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>compile</phase>
+                <goals>
+                  <goal>javah</goal>
+                </goals>
+                <configuration>
+                  <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+                  <javahClassNames>
+                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibCompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsMapping</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.nativeio.NativeIO</javahClassName>
+                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyCompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
+                  </javahClassNames>
+                  <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>compile-ms-winutils</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/winutils/winutils.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+              <execution>
+                <id>compile-ms-native-dll</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/native/native.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 </project>

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop Thu May 23 20:41:53 2013
@@ -91,9 +91,6 @@ case $COMMAND in
     ;;
 
   classpath)
-    if $cygwin; then
-      CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-    fi
     echo $CLASSPATH
     exit
     ;;
@@ -132,9 +129,6 @@ case $COMMAND in
     #make sure security appender is turned off
     HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
 
-    if $cygwin; then
-      CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-    fi
     export CLASSPATH=$CLASSPATH
     exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
     ;;

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh Thu May 23 20:41:53 2013
@@ -108,11 +108,6 @@ if [[ ( "$HADOOP_SLAVES" != '' ) && ( "$
   exit 1
 fi
 
-cygwin=false
-case "`uname`" in
-CYGWIN*) cygwin=true;;
-esac
-
 if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
   . "${HADOOP_CONF_DIR}/hadoop-env.sh"
 fi
@@ -199,13 +194,6 @@ fi
 # restore ordinary behaviour
 unset IFS
 
-# cygwin path translation
-if $cygwin; then
-  HADOOP_PREFIX=`cygpath -w "$HADOOP_PREFIX"`
-  HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"`
-  JAVA_LIBRARY_PATH=`cygpath -w "$JAVA_LIBRARY_PATH"`
-fi
-
 # setup 'java.library.path' for native-hadoop code if necessary
 
 if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then
@@ -222,11 +210,6 @@ fi
 # setup a default TOOL_PATH
 TOOL_PATH="${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}"
 
-# cygwin path translation
-if $cygwin; then
-  JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`
-fi
-
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
@@ -306,12 +289,3 @@ if [ "$HADOOP_CLASSPATH" != "" ]; then
   fi
 fi
 
-# cygwin path translation
-if $cygwin; then
-  HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"`
-fi
-
-# cygwin path translation
-if $cygwin; then
-  TOOL_PATH=`cygpath -p -w "$TOOL_PATH"`
-fi

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/rcc
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/rcc?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/rcc (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/bin/rcc Thu May 23 20:41:53 2013
@@ -57,10 +57,5 @@ unset IFS
 
 CLASS='org.apache.hadoop.record.compiler.generated.Rcc'
 
-# cygwin path translation
-if expr `uname` : 'CYGWIN*' > /dev/null; then
-  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-
 # run it
 exec "$JAVA" $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml Thu May 23 20:41:53 2013
@@ -87,7 +87,6 @@ See http://forrest.apache.org/docs/linki
     <zlib      href="http://www.zlib.net/" />
     <gzip      href="http://www.gzip.org/" />
     <bzip      href="http://www.bzip.org/" />
-    <cygwin    href="http://www.cygwin.com/" />
     <osx       href="http://www.apple.com/macosx" />
     
     <relnotes href="releasenotes.html" />

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java Thu May 23 20:41:53 2013
@@ -35,7 +35,7 @@ import com.google.common.annotations.Vis
 
 /** Filesystem disk space usage statistics.
  * Uses the unix 'df' program to get mount points, and java.io.File for
- * space utilization. Tested on Linux, FreeBSD, Cygwin. */
+ * space utilization. Tested on Linux, FreeBSD, Windows. */
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
 public class DF extends Shell {
@@ -164,10 +164,23 @@ public class DF extends Shell {
   }
 
   @Override
+  protected void run() throws IOException {
+    if (WINDOWS) {
+      try {
+        this.mount = dirFile.getCanonicalPath().substring(0,2);
+      } catch (IOException e) {
+      }
+      return;
+    }
+    super.run();
+  }
+
+  @Override
   protected String[] getExecString() {
     // ignoring the error since the exit code it enough
-    return new String[] {"bash","-c","exec 'df' '-k' '-P' '" + dirPath 
-                         + "' 2>/dev/null"};
+    return (WINDOWS)? new String[]{"cmd", "/c", "df -k " + dirPath + " 2>nul"}:
+        new String[] {"bash","-c","exec 'df' '-k' '-P' '" + dirPath 
+                      + "' 2>/dev/null"};
   }
 
   @Override

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java Thu May 23 20:41:53 2013
@@ -145,6 +145,20 @@ public class DU extends Shell {
   public String getDirPath() {
     return dirPath;
   }
+
+
+  /**
+   * Override to hook in DUHelper class. Maybe this can be used more
+   * generally as well on Unix/Linux based systems
+   */
+  @Override
+  protected void run() throws IOException {
+    if (WINDOWS) {
+      used.set(DUHelper.getFolderUsage(dirPath));
+      return;
+    }
+    super.run();
+  }
   
   /**
    * Start the disk usage checking thread.

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Thu May 23 20:41:53 2013
@@ -19,15 +19,26 @@
 package org.apache.hadoop.fs;
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
+import org.apache.commons.collections.map.CaseInsensitiveMap;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
@@ -43,6 +54,13 @@ public class FileUtil {
 
   private static final Log LOG = LogFactory.getLog(FileUtil.class);
 
+  /* The error code is defined in winutils to indicate insufficient
+   * privilege to create symbolic links. This value need to keep in
+   * sync with the constant of the same name in:
+   * "src\winutils\common.h"
+   * */
+  public static final int SYMLINK_NO_PRIVILEGE = 2;
+
   /**
    * convert an array of FileStatus to an array of Path
    * 
@@ -467,45 +485,13 @@ public class FileUtil {
   }
 
   /**
-   * This class is only used on windows to invoke the cygpath command.
-   */
-  private static class CygPathCommand extends Shell {
-    String[] command;
-    String result;
-    CygPathCommand(String path) throws IOException {
-      command = new String[]{"cygpath", "-u", path};
-      run();
-    }
-    String getResult() throws IOException {
-      return result;
-    }
-    @Override
-    protected String[] getExecString() {
-      return command;
-    }
-    @Override
-    protected void parseExecResult(BufferedReader lines) throws IOException {
-      String line = lines.readLine();
-      if (line == null) {
-        throw new IOException("Can't convert '" + command[2] + 
-                              " to a cygwin path");
-      }
-      result = line;
-    }
-  }
-
-  /**
    * Convert a os-native filename to a path that works for the shell.
    * @param filename The filename to convert
    * @return The unix pathname
    * @throws IOException on windows, there can be problems with the subprocess
    */
   public static String makeShellPath(String filename) throws IOException {
-    if (Path.WINDOWS) {
-      return new CygPathCommand(filename).getResult();
-    } else {
-      return filename;
-    }    
+    return filename;
   }
   
   /**
@@ -641,7 +627,7 @@ public class FileUtil {
     untarCommand.append(FileUtil.makeShellPath(untarDir)); 
     untarCommand.append("' ; ");
     untarCommand.append("tar -xf ");
-    
+
     if (gzipped) {
       untarCommand.append(" -)");
     } else {
@@ -659,7 +645,7 @@ public class FileUtil {
 
   /**
    * Class for creating hardlinks.
-   * Supports Unix, Cygwin, WindXP.
+   * Supports Unix, WindXP.
    * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
    */
   @Deprecated
@@ -671,21 +657,67 @@ public class FileUtil {
 
   /**
    * Create a soft link between a src and destination
-   * only on a local disk. HDFS does not support this
+   * only on a local disk. HDFS does not support this.
+   * On Windows, when symlink creation fails due to security
+   * setting, we will log a warning. The return code in this
+   * case is 2.
    * @param target the target for symlink 
    * @param linkname the symlink
    * @return value returned by the command
    */
   public static int symLink(String target, String linkname) throws IOException{
-    String cmd = "ln -s " + target + " " + linkname;
-    Process p = Runtime.getRuntime().exec(cmd, null);
-    int returnVal = -1;
-    try{
-      returnVal = p.waitFor();
-    } catch(InterruptedException e){
-      //do nothing as of yet
+    // Run the input paths through Java's File so that they are converted to the
+    // native OS form
+    File targetFile = new File(target);
+    File linkFile = new File(linkname);
+
+    // If not on Java7+, copy a file instead of creating a symlink since
+    // Java6 has close to no support for symlinks on Windows. Specifically
+    // File#length and File#renameTo do not work as expected.
+    // (see HADOOP-9061 for additional details)
+    // We still create symlinks for directories, since the scenario in this
+    // case is different. The directory content could change in which
+    // case the symlink loses its purpose (for example task attempt log folder
+    // is symlinked under userlogs and userlogs are generated afterwards).
+    if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
+      try {
+        LOG.info("FileUtil#symlink: On Java6, copying file instead "
+            + linkname + " -> " + target);
+        org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
+      } catch (IOException ex) {
+        LOG.warn("FileUtil#symlink failed to copy the file with error: "
+            + ex.getMessage());
+        // Exit with non-zero exit code
+        return 1;
+      }
+      return 0;
     }
-    return returnVal;
+
+    String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
+        linkFile.getPath());
+    ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
+    try {
+      shExec.execute();
+    } catch (Shell.ExitCodeException ec) {
+      int returnVal = ec.getExitCode();
+      if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
+        LOG.warn("Fail to create symbolic links on Windows. "
+            + "The default security settings in Windows disallow non-elevated "
+            + "administrators and all non-administrators from creating symbolic links. "
+            + "This behavior can be changed in the Local Security Policy management console");
+      } else if (returnVal != 0) {
+        LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
+            + returnVal + " with: " + ec.getMessage());
+      }
+      return returnVal;
+    } catch (IOException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error while create symlink " + linkname + " to " + target
+            + "." + " Exception: " + StringUtils.stringifyException(e));
+      }
+      throw e;
+    }
+    return shExec.getExitCode();
   }
   
   /**
@@ -709,30 +741,120 @@ public class FileUtil {
    * @param recursive true, if permissions should be changed recursively
    * @return the exit code from the command.
    * @throws IOException
-   * @throws InterruptedException
    */
   public static int chmod(String filename, String perm, boolean recursive)
-                            throws IOException, InterruptedException {
-    StringBuilder cmdBuf = new StringBuilder();
-    cmdBuf.append("chmod ");
-    if (recursive) {
-      cmdBuf.append("-R ");
-    }
-    cmdBuf.append(perm).append(" ");
-    cmdBuf.append(filename);
-    String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
-    ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
+                            throws IOException {
+    String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = new File(filename).getPath();
+    ShellCommandExecutor shExec = new ShellCommandExecutor(args);
     try {
       shExec.execute();
-    }catch(Exception e) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Error while changing permission : " + filename
-            + " Exception: ", e);
+    }catch(IOException e) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Error while changing permission : " + filename 
+                  +" Exception: " + StringUtils.stringifyException(e));
       }
     }
     return shExec.getExitCode();
   }
+
+  /**
+   * Set the ownership on a file / directory. User name and group name
+   * cannot both be null.
+   * @param file the file to change
+   * @param username the new user owner name
+   * @param groupname the new group owner name
+   * @throws IOException
+   */
+  public static void setOwner(File file, String username,
+      String groupname) throws IOException {
+    if (username == null && groupname == null) {
+      throw new IOException("username == null && groupname == null");
+    }
+    String arg = (username == null ? "" : username)
+        + (groupname == null ? "" : ":" + groupname);
+    String [] cmd = Shell.getSetOwnerCommand(arg);
+    execCommand(file, cmd);
+  }
+
+  /**
+   * Set permissions to the required value. Uses the java primitives instead
+   * of forking if group == other.
+   * @param f the file to change
+   * @param permission the new permissions
+   * @throws IOException
+   */
+  public static void setPermission(File f, FsPermission permission
+                                   ) throws IOException {
+    FsAction user = permission.getUserAction();
+    FsAction group = permission.getGroupAction();
+    FsAction other = permission.getOtherAction();
+
+    // use the native/fork if the group/other permissions are different
+    // or if the native is available or on Windows
+    if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
+      execSetPermission(f, permission);
+      return;
+    }
+    
+    boolean rv = true;
+    
+    // read perms
+    rv = f.setReadable(group.implies(FsAction.READ), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
+      rv = f.setReadable(user.implies(FsAction.READ), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // write perms
+    rv = f.setWritable(group.implies(FsAction.WRITE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
+      rv = f.setWritable(user.implies(FsAction.WRITE), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // exec perms
+    rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
+      rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
+      checkReturnValue(rv, f, permission);
+    }
+  }
+
+  private static void checkReturnValue(boolean rv, File p, 
+                                       FsPermission permission
+                                       ) throws IOException {
+    if (!rv) {
+      throw new IOException("Failed to set permissions of path: " + p + 
+                            " to " + 
+                            String.format("%04o", permission.toShort()));
+    }
+  }
+  
+  private static void execSetPermission(File f, 
+                                        FsPermission permission
+                                       )  throws IOException {
+    if (NativeIO.isAvailable()) {
+      NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
+    } else {
+      execCommand(f, Shell.getSetPermissionCommand(
+                  String.format("%04o", permission.toShort()), false));
+    }
+  }
   
+  static String execCommand(File f, String... cmd) throws IOException {
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = f.getCanonicalPath();
+    String output = Shell.execCommand(args);
+    return output;
+  }
+
   /**
    * Create a tmp file for a base file.
    * @param basefile the base file of the tmp
@@ -820,4 +942,97 @@ public class FileUtil {
     }
     return fileNames;
   }  
+  
+  /**
+   * Create a jar file at the given path, containing a manifest with a classpath
+   * that references all specified entries.
+   * 
+   * Some platforms may have an upper limit on command line length.  For example,
+   * the maximum command line length on Windows is 8191 characters, but the
+   * length of the classpath may exceed this.  To work around this limitation,
+   * use this method to create a small intermediate jar with a manifest that
+   * contains the full classpath.  It returns the absolute path to the new jar,
+   * which the caller may set as the classpath for a new process.
+   * 
+   * Environment variable evaluation is not supported within a jar manifest, so
+   * this method expands environment variables before inserting classpath entries
+   * to the manifest.  The method parses environment variables according to
+   * platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
+   * environment variables are case-insensitive.  For example, %VAR% and %var%
+   * evaluate to the same value.
+   * 
+   * Specifying the classpath in a jar manifest does not support wildcards, so
+   * this method expands wildcards internally.  Any classpath entry that ends
+   * with * is translated to all files at that path with extension .jar or .JAR.
+   * 
+   * @param inputClassPath String input classpath to bundle into the jar manifest
+   * @param pwd Path to working directory to save jar
+   * @return String absolute path to new jar
+   * @throws IOException if there is an I/O error while writing the jar file
+   */
+  public static String createJarWithClassPath(String inputClassPath, Path pwd)
+      throws IOException {
+    // Replace environment variables, case-insensitive on Windows
+    @SuppressWarnings("unchecked")
+    Map<String, String> env = Shell.WINDOWS ?
+      new CaseInsensitiveMap(System.getenv()) : System.getenv();
+    String[] classPathEntries = inputClassPath.split(File.pathSeparator);
+    for (int i = 0; i < classPathEntries.length; ++i) {
+      classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
+        StringUtils.ENV_VAR_PATTERN, env);
+    }
+    File workingDir = new File(pwd.toString());
+    if (!workingDir.mkdirs()) {
+      // If mkdirs returns false because the working directory already exists,
+      // then this is acceptable.  If it returns false due to some other I/O
+      // error, then this method will fail later with an IOException while saving
+      // the jar.
+      LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
+    }
+
+    // Append all entries
+    List<String> classPathEntryList = new ArrayList<String>(
+      classPathEntries.length);
+    for (String classPathEntry: classPathEntries) {
+      if (classPathEntry.endsWith("*")) {
+        // Append all jars that match the wildcard
+        Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
+        FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
+          .globStatus(globPath);
+        if (wildcardJars != null) {
+          for (FileStatus wildcardJar: wildcardJars) {
+            classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
+              .toExternalForm());
+          }
+        }
+      } else {
+        // Append just this jar
+        classPathEntryList.add(new File(classPathEntry).toURI().toURL()
+          .toExternalForm());
+      }
+    }
+    String jarClassPath = StringUtils.join(" ", classPathEntryList);
+
+    // Create the manifest
+    Manifest jarManifest = new Manifest();
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.CLASS_PATH.toString(), jarClassPath);
+
+    // Write the manifest to output JAR file
+    File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
+    FileOutputStream fos = null;
+    BufferedOutputStream bos = null;
+    JarOutputStream jos = null;
+    try {
+      fos = new FileOutputStream(classPathJar);
+      bos = new BufferedOutputStream(fos);
+      jos = new JarOutputStream(bos, jarManifest);
+    } finally {
+      IOUtils.cleanup(LOG, jos, bos, fos);
+    }
+
+    return classPathJar.getCanonicalPath();
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java Thu May 23 20:41:53 2013
@@ -25,9 +25,11 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Arrays;
 
+import org.apache.hadoop.util.Shell;
+
 /**
  * Class for creating hardlinks.
- * Supports Unix/Linux, WinXP/2003/Vista via Cygwin, and Mac OS X.
+ * Supports Unix/Linux, Windows via winutils , and Mac OS X.
  * 
  * The HardLink class was formerly a static inner class of FSUtil,
  * and the methods provided were blatantly non-thread-safe.
@@ -41,7 +43,7 @@ public class HardLink { 
 
   public enum OSType {
     OS_TYPE_UNIX,
-    OS_TYPE_WINXP,
+    OS_TYPE_WIN,
     OS_TYPE_SOLARIS,
     OS_TYPE_MAC,
     OS_TYPE_FREEBSD
@@ -56,7 +58,7 @@ public class HardLink { 
   //methods without instantiating the HardLink object
   static { 
     osType = getOSType();
-    if (osType == OSType.OS_TYPE_WINXP) {
+    if (osType == OSType.OS_TYPE_WIN) {
       // Windows
       getHardLinkCommand = new HardLinkCGWin();
     } else {
@@ -80,14 +82,8 @@ public class HardLink { 
   
   static private OSType getOSType() {
     String osName = System.getProperty("os.name");
-    if (osName.contains("Windows") &&
-            (osName.contains("XP") 
-            || osName.contains("2003") 
-            || osName.contains("Vista")
-            || osName.contains("Windows_7")
-            || osName.contains("Windows 7") 
-            || osName.contains("Windows7"))) {
-      return OSType.OS_TYPE_WINXP;
+    if (Shell.WINDOWS) {
+      return OSType.OS_TYPE_WIN;
     }
     else if (osName.contains("SunOS") 
             || osName.contains("Solaris")) {
@@ -258,11 +254,6 @@ public class HardLink { 
   
   /**
    * Implementation of HardLinkCommandGetter class for Windows
-   * 
-   * Note that the linkCount shell command for Windows is actually
-   * a Cygwin shell command, and depends on ${cygwin}/bin
-   * being in the Windows PATH environment variable, so
-   * stat.exe can be found.
    */
   static class HardLinkCGWin extends HardLinkCommandGetter {
     //The Windows command getter impl class and its member fields are
@@ -270,14 +261,16 @@ public class HardLink { 
     //unit testing (sort of) on non-Win servers
 
     static String[] hardLinkCommand = {
-                        "fsutil","hardlink","create", null, null};
+                        Shell.WINUTILS,"hardlink","create", null, null};
     static String[] hardLinkMultPrefix = {
                         "cmd","/q","/c","for", "%f", "in", "("};
     static String   hardLinkMultDir = "\\%f";
     static String[] hardLinkMultSuffix = {
-                        ")", "do", "fsutil", "hardlink", "create", null, 
+                        ")", "do", Shell.WINUTILS, "hardlink", "create", null,
                         "%f", "1>NUL"};
-    static String[] getLinkCountCommand = {"stat","-c%h", null};
+    static String[] getLinkCountCommand = {
+                        Shell.WINUTILS, "hardlink",
+                        "stat", null};
     //Windows guarantees only 8K - 1 bytes cmd length.
     //Subtract another 64b to allow for Java 'exec' overhead
     static final int maxAllowedCmdArgLength = 8*1024 - 65;
@@ -328,12 +321,6 @@ public class HardLink { 
       String[] buf = new String[getLinkCountCommand.length];
       System.arraycopy(getLinkCountCommand, 0, buf, 0, 
                        getLinkCountCommand.length);
-      //The linkCount command is actually a Cygwin shell command,
-      //not a Windows shell command, so we should use "makeShellPath()"
-      //instead of "getCanonicalPath()".  However, that causes another
-      //shell exec to "cygpath.exe", and "stat.exe" actually can handle
-      //DOS-style paths (it just prints a couple hundred bytes of warning
-      //to stderr), so we use the more efficient "getCanonicalPath()".
       buf[getLinkCountCommand.length - 1] = file.getCanonicalPath();
       return buf;
     }
@@ -354,7 +341,7 @@ public class HardLink { 
       //add the fixed overhead of the hardLinkMult command 
       //(prefix, suffix, and Dir suffix)
       sum += ("cmd.exe /q /c for %f in ( ) do "
-              + "fsutil hardlink create \\%f %f 1>NUL ").length();
+              + Shell.WINUTILS + " hardlink create \\%f %f 1>NUL ").length();
       return sum;
     }
     
@@ -581,14 +568,10 @@ public class HardLink { 
   /* Create an IOException for failing to get link count. */
   private static IOException createIOException(File f, String message,
       String error, int exitvalue, Exception cause) {
-    
-    final String winErrMsg = "; Windows errors in getLinkCount are often due "
-         + "to Cygwin misconfiguration";
 
     final String s = "Failed to get link count on file " + f
         + ": message=" + message
         + "; error=" + error
-        + ((osType == OSType.OS_TYPE_WINXP) ? winErrMsg : "")
         + "; exit value=" + exitvalue;
     return (cause == null) ? new IOException(s) : new IOException(s, cause);
   }

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Thu May 23 20:41:53 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.regex.Pattern;
 
 import org.apache.avro.reflect.Stringable;
 import org.apache.commons.lang.StringUtils;
@@ -43,9 +44,17 @@ public class Path implements Comparable 
   
   public static final String CUR_DIR = ".";
   
-  static final boolean WINDOWS
+  public static final boolean WINDOWS
     = System.getProperty("os.name").startsWith("Windows");
 
+  /**
+   *  Pre-compiled regular expressions to detect path formats.
+   */
+  private static final Pattern hasUriScheme =
+      Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+  private static final Pattern hasDriveLetterSpecifier =
+      Pattern.compile("^/?[a-zA-Z]:");
+
   private URI uri;                                // a hierarchical uri
 
   /** Resolve a child path against a parent path. */
@@ -81,7 +90,7 @@ public class Path implements Comparable 
                resolved.getPath(), resolved.getFragment());
   }
 
-  private void checkPathArg( String path ) {
+  private void checkPathArg( String path ) throws IllegalArgumentException {
     // disallow construction of a Path from an empty string
     if ( path == null ) {
       throw new IllegalArgumentException(
@@ -95,15 +104,16 @@ public class Path implements Comparable 
   
   /** Construct a path from a String.  Path strings are URIs, but with
    * unescaped elements and some additional normalization. */
-  public Path(String pathString) {
+  public Path(String pathString) throws IllegalArgumentException {
     checkPathArg( pathString );
     
     // We can't use 'new URI(String)' directly, since it assumes things are
     // escaped, which we don't require of Paths. 
     
     // add a slash in front of paths with Windows drive letters
-    if (hasWindowsDrive(pathString, false))
-      pathString = "/"+pathString;
+    if (hasWindowsDrive(pathString) && pathString.charAt(0) != '/') {
+      pathString = "/" + pathString;
+    }
 
     // parse uri components
     String scheme = null;
@@ -151,22 +161,54 @@ public class Path implements Comparable 
   private void initialize(String scheme, String authority, String path,
       String fragment) {
     try {
-      this.uri = new URI(scheme, authority, normalizePath(path), null, fragment)
+      this.uri = new URI(scheme, authority, normalizePath(scheme, path), null, fragment)
         .normalize();
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
     }
   }
 
-  private String normalizePath(String path) {
-    // remove double slashes & backslashes
+  /**
+   * Merge 2 paths such that the second path is appended relative to the first.
+   * The returned path has the scheme and authority of the first path.  On
+   * Windows, the drive specification in the second path is discarded.
+   * 
+   * @param path1 Path first path
+   * @param path2 Path second path, to be appended relative to path1
+   * @return Path merged path
+   */
+  public static Path mergePaths(Path path1, Path path2) {
+    String path2Str = path2.toUri().getPath();
+    if(hasWindowsDrive(path2Str)) {
+      path2Str = path2Str.substring(path2Str.indexOf(':')+1);
+    }
+    return new Path(path1 + path2Str);
+  }
+
+  /**
+   * Normalize a path string to use non-duplicated forward slashes as
+   * the path separator and remove any trailing path separators.
+   * @param scheme Supplies the URI scheme. Used to deduce whether we
+   *               should replace backslashes or not.
+   * @param path Supplies the scheme-specific part
+   * @return Normalized path string.
+   */
+  private static String normalizePath(String scheme, String path) {
+    // Remove double forward slashes.
     path = StringUtils.replace(path, "//", "/");
-    if (Path.WINDOWS) {
+
+    // Remove backslashes if this looks like a Windows path. Avoid
+    // the substitution if it looks like a non-local URI.
+    if (WINDOWS &&
+        (hasWindowsDrive(path) ||
+         (scheme == null) ||
+         (scheme.isEmpty()) ||
+         (scheme.equals("file")))) {
       path = StringUtils.replace(path, "\\", "/");
     }
     
     // trim trailing slash from non-root path (ignoring windows drive)
-    int minLength = hasWindowsDrive(path, true) ? 4 : 1;
+    int minLength = hasWindowsDrive(path) ? 4 : 1;
     if (path.length() > minLength && path.endsWith("/")) {
       path = path.substring(0, path.length()-1);
     }
@@ -174,17 +216,29 @@ public class Path implements Comparable 
     return path;
   }
 
-  private boolean hasWindowsDrive(String path, boolean slashed) {
-    if (!WINDOWS) return false;
-    int start = slashed ? 1 : 0;
-    return
-      path.length() >= start+2 &&
-      (slashed ? path.charAt(0) == '/' : true) &&
-      path.charAt(start+1) == ':' &&
-      ((path.charAt(start) >= 'A' && path.charAt(start) <= 'Z') ||
-       (path.charAt(start) >= 'a' && path.charAt(start) <= 'z'));
+  private static boolean hasWindowsDrive(String path) {
+    return (WINDOWS && hasDriveLetterSpecifier.matcher(path).find());
   }
 
+  /**
+   * Determine whether a given path string represents an absolute path on
+   * Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+   *
+   * @param pathString Supplies the path string to evaluate.
+   * @param slashed true if the given path is prefixed with "/".
+   * @return true if the supplied path looks like an absolute path with a Windows
+   * drive-specifier.
+   */
+  public static boolean isWindowsAbsolutePath(final String pathString,
+                                              final boolean slashed) {
+    int start = (slashed ? 1 : 0);
+
+    return
+        hasWindowsDrive(pathString) &&
+        pathString.length() >= (start + 3) &&
+        ((pathString.charAt(start + 2) == SEPARATOR_CHAR) ||
+          (pathString.charAt(start + 2) == '\\'));
+  }
 
   /** Convert this to a URI. */
   public URI toUri() { return uri; }
@@ -207,7 +261,7 @@ public class Path implements Comparable 
    *  True if the path component (i.e. directory) of this URI is absolute.
    */
   public boolean isUriPathAbsolute() {
-    int start = hasWindowsDrive(uri.getPath(), true) ? 3 : 0;
+    int start = hasWindowsDrive(uri.getPath()) ? 3 : 0;
     return uri.getPath().startsWith(SEPARATOR, start);
    }
   
@@ -241,7 +295,7 @@ public class Path implements Comparable 
   public Path getParent() {
     String path = uri.getPath();
     int lastSlash = path.lastIndexOf('/');
-    int start = hasWindowsDrive(path, true) ? 3 : 0;
+    int start = hasWindowsDrive(path) ? 3 : 0;
     if ((path.length() == start) ||               // empty path
         (lastSlash == start && path.length() == start+1)) { // at root
       return null;
@@ -250,7 +304,7 @@ public class Path implements Comparable 
     if (lastSlash==-1) {
       parent = CUR_DIR;
     } else {
-      int end = hasWindowsDrive(path, true) ? 3 : 0;
+      int end = hasWindowsDrive(path) ? 3 : 0;
       parent = path.substring(0, lastSlash==end?end+1:lastSlash);
     }
     return new Path(uri.getScheme(), uri.getAuthority(), parent);
@@ -277,7 +331,7 @@ public class Path implements Comparable 
     if (uri.getPath() != null) {
       String path = uri.getPath();
       if (path.indexOf('/')==0 &&
-          hasWindowsDrive(path, true) &&          // has windows drive
+          hasWindowsDrive(path) &&                // has windows drive
           uri.getScheme() == null &&              // but no scheme
           uri.getAuthority() == null)             // or authority
         path = path.substring(1);                 // remove slash before drive
@@ -364,7 +418,7 @@ public class Path implements Comparable 
     URI newUri = null;
     try {
       newUri = new URI(scheme, authority , 
-        normalizePath(pathUri.getPath()), null, fragment);
+        normalizePath(scheme, pathUri.getPath()), null, fragment);
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
     }

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Thu May 23 20:41:53 2013
@@ -502,9 +502,10 @@ public class RawLocalFileSystem extends 
       return !super.getOwner().equals(""); 
     }
     
-    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
+    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { 
       super(f.length(), f.isDirectory(), 1, defaultBlockSize,
-            f.lastModified(), fs.makeQualified(new Path(f.getPath())));
+          f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(),
+            fs.getWorkingDirectory()));
     }
     
     @Override
@@ -535,9 +536,10 @@ public class RawLocalFileSystem extends 
     private void loadPermissionInfo() {
       IOException e = null;
       try {
-        StringTokenizer t = new StringTokenizer(
-            execCommand(new File(getPath().toUri()), 
-                        Shell.getGET_PERMISSION_COMMAND()));
+        String output = FileUtil.execCommand(new File(getPath().toUri()), 
+            Shell.getGetPermissionCommand());
+        StringTokenizer t =
+            new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
         //expected format
         //-rw-------    1 username groupname ...
         String permission = t.nextToken();
@@ -546,7 +548,17 @@ public class RawLocalFileSystem extends 
         }
         setPermission(FsPermission.valueOf(permission));
         t.nextToken();
-        setOwner(t.nextToken());
+
+        String owner = t.nextToken();
+        // If on windows domain, token format is DOMAIN\\user and we want to
+        // extract only the user name
+        if (Shell.WINDOWS) {
+          int i = owner.indexOf('\\');
+          if (i != -1)
+            owner = owner.substring(i + 1);
+        }
+        setOwner(owner);
+
         setGroup(t.nextToken());
       } catch (Shell.ExitCodeException ioe) {
         if (ioe.getExitCode() != 1) {
@@ -582,17 +594,7 @@ public class RawLocalFileSystem extends 
   @Override
   public void setOwner(Path p, String username, String groupname)
     throws IOException {
-    if (username == null && groupname == null) {
-      throw new IOException("username == null && groupname == null");
-    }
-
-    if (username == null) {
-      execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
-    } else {
-      //OWNER[:[GROUP]]
-      String s = username + (groupname == null? "": ":" + groupname);
-      execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
-    }
+    FileUtil.setOwner(pathToFile(p), username, groupname);
   }
 
   /**
@@ -602,11 +604,12 @@ public class RawLocalFileSystem extends 
   public void setPermission(Path p, FsPermission permission)
     throws IOException {
     if (NativeIO.isAvailable()) {
-      NativeIO.chmod(pathToFile(p).getCanonicalPath(),
+      NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
                      permission.toShort());
     } else {
-      execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
-          String.format("%05o", permission.toShort()));
+      String perm = String.format("%04o", permission.toShort());
+      Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
+        FileUtil.makeShellPath(pathToFile(p), true)));
     }
   }
 
@@ -640,4 +643,4 @@ public class RawLocalFileSystem extends 
     return output;
   }
 
-}
\ No newline at end of file
+}

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java Thu May 23 20:41:53 2013
@@ -92,7 +92,7 @@ public class TrashPolicyDefault extends 
   }
   
   private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
-    return new Path(basePath + rmFilePath.toUri().getPath());
+    return Path.mergePaths(basePath, rmFilePath);
   }
 
   @Override

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java Thu May 23 20:41:53 2013
@@ -89,11 +89,9 @@ public class RawLocalFs extends Delegate
     }
     // NB: Use createSymbolicLink in java.nio.file.Path once available
     try {
-      Shell.execCommand(Shell.LINK_COMMAND, "-s",
-                        new URI(target.toString()).getPath(),
-                        new URI(link.toString()).getPath());
-    } catch (URISyntaxException x) {
-      throw new IOException("Invalid symlink path: "+x.getMessage());
+      Shell.execCommand(Shell.getSymlinkCommand(
+        getPathWithoutSchemeAndAuthority(target),
+        getPathWithoutSchemeAndAuthority(link)));
     } catch (IOException x) {
       throw new IOException("Unable to create symlink: "+x.getMessage());
     }
@@ -168,4 +166,13 @@ public class RawLocalFs extends Delegate
      */
     throw new AssertionError();
   }
+
+  private static String getPathWithoutSchemeAndAuthority(Path path) {
+    // This code depends on Path.toString() to remove the leading slash before
+    // the drive specification on Windows.
+    Path newPath = path.isUriPathAbsolute() ?
+      new Path(null, null, path.toUri().getPath()) :
+      path;
+    return newPath.toString();
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Thu May 23 20:41:53 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs.shell;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.LinkedList;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -83,8 +85,12 @@ abstract class CommandWithDestination ex
    */
   protected void getLocalDestination(LinkedList<String> args)
   throws IOException {
-    String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
-    dst = new PathData(new File(pathString), getConf());
+    try {
+      String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
+      dst = new PathData(new URI(pathString), getConf());
+    } catch (URISyntaxException e) {
+      throw new IOException("unexpected URISyntaxException", e);
+    }
   }
 
   /**
@@ -319,4 +325,4 @@ abstract class CommandWithDestination ex
       processDeleteOnExit();
     }
   }
-}
\ No newline at end of file
+}

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Thu May 23 20:41:53 2013
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.shell;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -60,16 +62,20 @@ class CopyCommands {  
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
-      cf.parse(args);
+      try {
+        CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
+        cf.parse(args);
 
-      delimiter = cf.getOpt("nl") ? "\n" : null;
+        delimiter = cf.getOpt("nl") ? "\n" : null;
 
-      dst = new PathData(new File(args.removeLast()), getConf());
-      if (dst.exists && dst.stat.isDirectory()) {
-        throw new PathIsDirectoryException(dst.toString());
+        dst = new PathData(new URI(args.removeLast()), getConf());
+        if (dst.exists && dst.stat.isDirectory()) {
+          throw new PathIsDirectoryException(dst.toString());
+        }
+        srcs = new LinkedList<PathData>();
+      } catch (URISyntaxException e) {
+        throw new IOException("unexpected URISyntaxException", e);
       }
-      srcs = new LinkedList<PathData>();
     }
 
     @Override
@@ -197,9 +203,13 @@ class CopyCommands {  
     // commands operating on local paths have no need for glob expansion
     @Override
     protected List<PathData> expandArgument(String arg) throws IOException {
-      List<PathData> items = new LinkedList<PathData>();
-      items.add(new PathData(new File(arg), getConf()));
-      return items;
+      try {
+        List<PathData> items = new LinkedList<PathData>();
+        items.add(new PathData(new URI(arg), getConf()));
+        return items;
+      } catch (URISyntaxException e) {
+        throw new IOException("unexpected URISyntaxException", e);
+      }
     }
 
     @Override

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Thu May 23 20:41:53 2013
@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Arrays;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -39,6 +40,9 @@ import org.apache.hadoop.fs.PathNotFound
 
 /**
  * Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
+ * PathData ensures that the returned path string will be the same as the
+ * one passed in during initialization (unlike Path objects which can
+ * modify the path string).
  * The stat field will be null if the path does not exist.
  */
 @InterfaceAudience.Private
@@ -51,6 +55,20 @@ public class PathData implements Compara
   public FileStatus stat;
   public boolean exists;
 
+  /* True if the URI scheme was not present in the pathString but inferred.
+   */
+  private boolean inferredSchemeFromPath = false;
+
+  /**
+   *  Pre-compiled regular expressions to detect path formats.
+   */
+  private static final Pattern potentialUri =
+      Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+  private static final Pattern windowsNonUriAbsolutePath1 =
+      Pattern.compile("^/?[a-zA-Z]:\\\\");
+  private static final Pattern windowsNonUriAbsolutePath2 =
+      Pattern.compile("^/?[a-zA-Z]:/");
+
   /**
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
@@ -67,12 +85,12 @@ public class PathData implements Compara
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
    * return exactly the same string used to initialize it
-   * @param localPath a local File
+   * @param localPath a local URI
    * @param conf the configuration file
    * @throws IOException if anything goes wrong...
    */
-  public PathData(File localPath, Configuration conf) throws IOException {
-    this(FileSystem.getLocal(conf), localPath.toString());
+  public PathData(URI localPath, Configuration conf) throws IOException {
+    this(FileSystem.getLocal(conf), localPath.getPath());
   }
 
   /**
@@ -87,6 +105,39 @@ public class PathData implements Compara
   }
 
   /**
+   * Validates the given Windows path.
+   * Throws IOException on failure.
+   * @param pathString a String of the path suppliued by the user.
+   */
+  private void ValidateWindowsPath(String pathString)
+  throws IOException
+  {
+    if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+      // Forward slashes disallowed in a backslash-separated path.
+      if (pathString.indexOf('/') != -1) {
+        throw new IOException("Invalid path string " + pathString);
+      }
+
+      inferredSchemeFromPath = true;
+      return;
+    }
+
+    // Is it a forward slash-separated absolute path?
+    if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+      inferredSchemeFromPath = true;
+      return;
+    }
+
+    // Does it look like a URI? If so then just leave it alone.
+    if (potentialUri.matcher(pathString).find()) {
+      return;
+    }
+
+    // Looks like a relative path on Windows.
+    return;
+  }
+
+  /**
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
    * return exactly the same string used to initialize it.
@@ -100,6 +151,10 @@ public class PathData implements Compara
     this.uri = stringToUri(pathString);
     this.path = fs.makeQualified(new Path(uri));
     setStat(stat);
+
+    if (Path.WINDOWS) {
+      ValidateWindowsPath(pathString);
+    }
   }
 
   // need a static method for the ctor above
@@ -236,7 +291,7 @@ public class PathData implements Compara
    * Given a child of this directory, use the directory's path and the child's
    * basename to construct the string to the child.  This preserves relative
    * paths since Path will fully qualify.
-   * @param child a path contained within this directory
+   * @param childPath a path contained within this directory
    * @return String of the path relative to this directory
    */
   private String getStringForChildPath(Path childPath) {
@@ -386,7 +441,14 @@ public class PathData implements Compara
     // No interpretation of symbols. Just decode % escaped chars.
     String decodedRemainder = uri.getSchemeSpecificPart();
 
-    if (scheme == null) {
+    // Drop the scheme if it was inferred to ensure fidelity between
+    // the input and output path strings.
+    if ((scheme == null) || (inferredSchemeFromPath)) {
+      if (Path.isWindowsAbsolutePath(decodedRemainder, true)) {
+        // Strip the leading '/' added in stringToUri so users see a valid
+        // Windows path.
+        decodedRemainder = decodedRemainder.substring(1);
+      }
       return decodedRemainder;
     } else {
       StringBuilder buffer = new StringBuilder();
@@ -409,13 +471,56 @@ public class PathData implements Compara
     return ((LocalFileSystem)fs).pathToFile(path);
   }
 
+  /** Normalize the given Windows path string. This does the following:
+   *    1. Adds "file:" scheme for absolute paths.
+   *    2. Ensures the scheme-specific part starts with '/' per RFC2396.
+   *    3. Replaces backslash path separators with forward slashes.
+   *    @param pathString Path string supplied by the user.
+   *    @return normalized absolute path string. Returns the input string
+   *            if it is not a Windows absolute path.
+   */
+  private static String normalizeWindowsPath(String pathString)
+  throws IOException
+  {
+    if (!Path.WINDOWS) {
+      return pathString;
+    }
+
+    boolean slashed =
+        ((pathString.length() >= 1) && (pathString.charAt(0) == '/'));
+
+    // Is it a backslash-separated absolute path?
+    if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+      // Forward slashes disallowed in a backslash-separated path.
+      if (pathString.indexOf('/') != -1) {
+        throw new IOException("Invalid path string " + pathString);
+      }
+
+      pathString = pathString.replace('\\', '/');
+      return "file:" + (slashed ? "" : "/") + pathString;
+    }
+
+    // Is it a forward slash-separated absolute path?
+    if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+      return "file:" + (slashed ? "" : "/") + pathString;
+    }
+
+    // Is it a backslash-separated relative file path (no scheme and
+    // no drive-letter specifier)?
+    if ((pathString.indexOf(':') == -1) && (pathString.indexOf('\\') != -1)) {
+      pathString = pathString.replace('\\', '/');
+    }
+
+    return pathString;
+  }
+
   /** Construct a URI from a String with unescaped special characters
-   *  that have non-standard sematics. e.g. /, ?, #. A custom parsing
-   *  is needed to prevent misbihaviors.
+   *  that have non-standard semantics. e.g. /, ?, #. A custom parsing
+   *  is needed to prevent misbehavior.
    *  @param pathString The input path in string form
    *  @return URI
    */
-  private static URI stringToUri(String pathString) {
+  private static URI stringToUri(String pathString) throws IOException {
     // We can't use 'new URI(String)' directly. Since it doesn't do quoting
     // internally, the internal parser may fail or break the string at wrong
     // places. Use of multi-argument ctors will quote those chars for us,
@@ -424,9 +529,10 @@ public class PathData implements Compara
     // parse uri components
     String scheme = null;
     String authority = null;
-
     int start = 0;
 
+    pathString = normalizeWindowsPath(pathString);
+
     // parse uri scheme, if any
     int colon = pathString.indexOf(':');
     int slash = pathString.indexOf('/');
@@ -445,8 +551,7 @@ public class PathData implements Compara
       authority = pathString.substring(start, authEnd);
       start = authEnd;
     }
-
-    // uri path is the rest of the string. ? or # are not interpreated,
+    // uri path is the rest of the string. ? or # are not interpreted,
     // but any occurrence of them will be quoted by the URI ctor.
     String path = pathString.substring(start, pathString.length());
 

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Thu May 23 20:41:53 2013
@@ -60,6 +60,7 @@ import org.apache.hadoop.security.authen
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
 import org.mortbay.io.Buffer;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Handler;
@@ -304,6 +305,13 @@ public class HttpServer implements Filte
     ret.setAcceptQueueSize(128);
     ret.setResolveNames(false);
     ret.setUseDirectBuffers(false);
+    if(Shell.WINDOWS) {
+      // result of setting the SO_REUSEADDR flag is different on Windows
+      // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
+      // without this 2 NN's can start on the same machine and listen on 
+      // the same port with indeterminate routing of incoming requests to them
+      ret.setReuseAddress(false);
+    }
     ret.setHeaderBufferSize(1024*64);
     return ret;
   }

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java Thu May 23 20:41:53 2013
@@ -203,8 +203,8 @@ public class ReadaheadPool {
       // It's also possible that we'll end up requesting readahead on some
       // other FD, which may be wasted work, but won't cause a problem.
       try {
-        NativeIO.posixFadviseIfPossible(fd, off, len,
-            NativeIO.POSIX_FADV_WILLNEED);
+        NativeIO.POSIX.posixFadviseIfPossible(fd, off, len,
+            NativeIO.POSIX.POSIX_FADV_WILLNEED);
       } catch (IOException ioe) {
         if (canceled) {
           // no big deal - the reader canceled the request and closed

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java Thu May 23 20:41:53 2013
@@ -23,6 +23,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -32,7 +33,7 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.io.nativeio.Errno;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.io.nativeio.NativeIOException;
-import org.apache.hadoop.io.nativeio.NativeIO.Stat;
+import org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat;
 import org.apache.hadoop.security.UserGroupInformation;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -127,7 +128,7 @@ public class SecureIOUtils {
     RandomAccessFile raf = new RandomAccessFile(f, mode);
     boolean success = false;
     try {
-      Stat stat = NativeIO.getFstat(raf.getFD());
+      Stat stat = NativeIO.POSIX.getFstat(raf.getFD());
       checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
           expectedGroup);
       success = true;
@@ -169,7 +170,7 @@ public class SecureIOUtils {
         rawFilesystem.open(new Path(file.getAbsolutePath()));
     boolean success = false;
     try {
-      Stat stat = NativeIO.getFstat(in.getFileDescriptor());
+      Stat stat = NativeIO.POSIX.getFstat(in.getFileDescriptor());
       checkStat(file, stat.getOwner(), stat.getGroup(), expectedOwner,
           expectedGroup);
       success = true;
@@ -214,7 +215,7 @@ public class SecureIOUtils {
     FileInputStream fis = new FileInputStream(f);
     boolean success = false;
     try {
-      Stat stat = NativeIO.getFstat(fis.getFD());
+      Stat stat = NativeIO.POSIX.getFstat(fis.getFD());
       checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
           expectedGroup);
       success = true;
@@ -260,35 +261,30 @@ public class SecureIOUtils {
     if (skipSecurity) {
       return insecureCreateForWrite(f, permissions);
     } else {
-      // Use the native wrapper around open(2)
-      try {
-        FileDescriptor fd = NativeIO.open(f.getAbsolutePath(),
-          NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL,
-          permissions);
-        return new FileOutputStream(fd);
-      } catch (NativeIOException nioe) {
-        if (nioe.getErrno() == Errno.EEXIST) {
-          throw new AlreadyExistsException(nioe);
-        }
-        throw nioe;
-      }
+      return NativeIO.getCreateForWriteFileOutputStream(f, permissions);
     }
   }
 
   private static void checkStat(File f, String owner, String group, 
       String expectedOwner, 
       String expectedGroup) throws IOException {
+    boolean success = true;
     if (expectedOwner != null &&
         !expectedOwner.equals(owner)) {
-      throw new IOException(
-        "Owner '" + owner + "' for path " + f + " did not match " +
-        "expected owner '" + expectedOwner + "'");
+      if (Path.WINDOWS) {
+        UserGroupInformation ugi =
+            UserGroupInformation.createRemoteUser(expectedOwner);
+        final String adminsGroupString = "Administrators";
+        success = owner.equals(adminsGroupString)
+            && Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString);
+      } else {
+        success = false;
+      }
     }
-    if (expectedGroup != null &&
-        !expectedGroup.equals(group)) {
+    if (!success) {
       throw new IOException(
-        "Group '" + group + "' for path " + f + " did not match " +
-        "expected group '" + expectedGroup + "'");
+          "Owner '" + owner + "' for path " + f + " did not match " +
+              "expected owner '" + expectedOwner + "'");
     }
   }
 



Mime
View raw message