hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1485845 - in /hadoop/common/branches/branch-2: ./ hadoop-assemblies/src/main/resources/assemblies/ hadoop-dist/ hadoop-project-dist/ hadoop-project/ hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/
Date Thu, 23 May 2013 20:41:56 GMT
Author: suresh
Date: Thu May 23 20:41:53 2013
New Revision: 1485845

URL: http://svn.apache.org/r1485845
Log:
HADOOP-8562. Merge r1453486 from trunk

Modified:
    hadoop/common/branches/branch-2/BUILDING.txt
    hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
    hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
    hadoop/common/branches/branch-2/hadoop-dist/pom.xml
    hadoop/common/branches/branch-2/hadoop-project-dist/pom.xml
    hadoop/common/branches/branch-2/hadoop-project/pom.xml
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java

Modified: hadoop/common/branches/branch-2/BUILDING.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/BUILDING.txt?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/BUILDING.txt (original)
+++ hadoop/common/branches/branch-2/BUILDING.txt Thu May 23 20:41:53 2013
@@ -138,3 +138,70 @@ Create a local staging version of the we
   $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
 
 ----------------------------------------------------------------------------------
+
+Building on Windows
+
+----------------------------------------------------------------------------------
+Requirements:
+
+* Windows System
+* JDK 1.6
+* Maven 3.0
+* Findbugs 1.3.9 (if running findbugs)
+* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS)
+* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
+* Windows SDK or Visual Studio 2010 Professional
+* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
+
+If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
+Do not use Visual Studio Express.  It does not support compiling for 64-bit,
+which is problematic if running a 64-bit system.  The Windows SDK is free to
+download here:
+
+http://www.microsoft.com/en-us/download/details.aspx?id=8279
+
+----------------------------------------------------------------------------------
+Building:
+
+Keep the source code tree in a short path to avoid running into problems related
+to Windows maximum path length limitation.  (For example, C:\hdc).
+
+Run builds from a Windows SDK Command Prompt.  (Start, All Programs,
+Microsoft Windows SDK v7.1, Windows SDK 7.1 Command Prompt.)
+
+JAVA_HOME must be set, and the path must not contain spaces.  If the full path
+would contain spaces, then use the Windows short path instead.
+
+You must set the Platform environment variable to either x64 or Win32 depending
+on whether you're running a 64-bit or 32-bit system.  Note that this is
+case-sensitive.  It must be "Platform", not "PLATFORM" or "platform".
+Environment variables on Windows are usually case-insensitive, but Maven treats
+them as case-sensitive.  Failure to set this environment variable correctly will
+cause msbuild to fail while building the native code in hadoop-common.
+
+set Platform=x64 (when building on a 64-bit system)
+set Platform=Win32 (when building on a 32-bit system)
+
+Several tests require that the user must have the Create Symbolic Links
+privilege.
+
+All Maven goals are the same as described above, with the addition of profile
+-Pnative-win to trigger building Windows native components.  The native
+components are required (not optional) on Windows.  For example:
+
+ * Run tests                 : mvn -Pnative-win test
+
+----------------------------------------------------------------------------------
+Building distributions:
+
+Create binary distribution with native code and with documentation:
+
+  $ mvn package -Pdist,native-win,docs -DskipTests -Dtar
+
+Create source distribution:
+
+  $ mvn package -Pnative-win,src -DskipTests
+
+Create source and binary distributions with native code and documentation:
+
+  $ mvn package -Pdist,native-win,docs,src -DskipTests -Dtar

Modified: hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
(original)
+++ hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
Thu May 23 20:41:53 2013
@@ -26,6 +26,9 @@
       <outputDirectory>/bin</outputDirectory>
       <excludes>
         <exclude>*.sh</exclude>
+        <exclude>*-config.cmd</exclude>
+        <exclude>start-*.cmd</exclude>
+        <exclude>stop-*.cmd</exclude>
       </excludes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -38,6 +41,7 @@
       <outputDirectory>/libexec</outputDirectory>
       <includes>
         <include>*-config.sh</include>
+        <include>*-config.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -46,9 +50,13 @@
       <outputDirectory>/sbin</outputDirectory>
       <includes>
         <include>*.sh</include>
+        <include>*.cmd</include>
       </includes>
       <excludes>
         <exclude>hadoop-config.sh</exclude>
+        <exclude>hadoop.cmd</exclude>
+        <exclude>hdfs.cmd</exclude>
+        <exclude>hadoop-config.cmd</exclude>
       </excludes>
       <fileMode>0755</fileMode>
     </fileSet>

Modified: hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
(original)
+++ hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
Thu May 23 20:41:53 2013
@@ -33,6 +33,7 @@
       <outputDirectory>bin</outputDirectory>
       <includes>
         <include>yarn</include>
+        <include>yarn.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -41,6 +42,7 @@
       <outputDirectory>libexec</outputDirectory>
       <includes>
         <include>yarn-config.sh</include>
+        <include>yarn-config.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -52,6 +54,8 @@
         <include>yarn-daemons.sh</include>
         <include>start-yarn.sh</include>
         <include>stop-yarn.sh</include>
+        <include>start-yarn.cmd</include>
+        <include>stop-yarn.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>

Modified: hadoop/common/branches/branch-2/hadoop-dist/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-dist/pom.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-dist/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-dist/pom.xml Thu May 23 20:41:53 2013
@@ -107,7 +107,7 @@
                         fi
                       }
 
-                      ROOT=`cd ${basedir}/..;pwd`
+                      ROOT=`cd ../..;pwd`
                       echo
                       echo "Current directory `pwd`"
                       echo
@@ -151,7 +151,8 @@
                         fi
                       }
 
-                      run tar czf hadoop-${project.version}.tar.gz hadoop-${project.version}
+                      run tar cf hadoop-${project.version}.tar hadoop-${project.version}
+                      run gzip hadoop-${project.version}.tar
                       echo
                       echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
                       echo

Modified: hadoop/common/branches/branch-2/hadoop-project-dist/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-project-dist/pom.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-project-dist/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-project-dist/pom.xml Thu May 23 20:41:53 2013
@@ -335,13 +335,7 @@
                   <target>
                     <!-- Using Unix script to preserve symlinks -->
                     <echo file="${project.build.directory}/dist-copynativelibs.sh">
-
-                      which cygpath 2&gt; /dev/null
-                      if [ $? = 1 ]; then
-                        BUILD_DIR="${project.build.directory}"
-                      else
-                        BUILD_DIR=`cygpath --unix '${project.build.directory}'`
-                      fi
+                      BUILD_DIR="${project.build.directory}"
                       TAR='tar cf -'
                       UNTAR='tar xfBp -'
                       LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib"
@@ -355,6 +349,13 @@
                           $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR)
                         fi
                       fi
+                      BIN_DIR="${BUILD_DIR}/bin"
+                      if [ -d $${BIN_DIR} ] ; then
+                        TARGET_BIN_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/bin"
+                        mkdir -p $${TARGET_BIN_DIR}
+                        cd $${BIN_DIR}
+                        $$TAR * | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
+                      fi
                     </echo>
                     <exec executable="sh" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-copynativelibs.sh"/>
@@ -372,15 +373,8 @@
                   <target if="tar">
                     <!-- Using Unix script to preserve symlinks -->
                     <echo file="${project.build.directory}/dist-maketar.sh">
-
-                      which cygpath 2&gt; /dev/null
-                      if [ $? = 1 ]; then
-                        BUILD_DIR="${project.build.directory}"
-                      else
-                        BUILD_DIR=`cygpath --unix '${project.build.directory}'`
-                      fi
-                      cd ${BUILD_DIR}
-                      tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version}
+                      cd "${project.build.directory}"
+                      tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                     </echo>
                     <exec executable="sh" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-maketar.sh"/>

Modified: hadoop/common/branches/branch-2/hadoop-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-project/pom.xml?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-project/pom.xml Thu May 23 20:41:53 2013
@@ -840,6 +840,8 @@
           <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
           <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
           <environmentVariables>
+            <!-- HADOOP_HOME required for tests on Windows to find winutils -->
+            <HADOOP_HOME>${basedir}/../../hadoop-common-project/hadoop-common/target</HADOOP_HOME>
             <LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${basedir}/../../hadoop-common-project/hadoop-common/target/native/target/usr/local/lib/</LD_LIBRARY_PATH>
             <MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
           </environmentVariables>
@@ -905,6 +907,28 @@
       </properties>
     </profile>
     <profile>
+      <id>native-win</id>
+      <activation>
+        <os>
+          <family>Windows</family>
+        </os>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <environmentVariables>
+                <!-- Specify where to look for the native DLL on Windows -->
+                <PATH>${env.PATH};${basedir}/../../hadoop-common-project/hadoop-common/target/bin</PATH>
+              </environmentVariables>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
       <id>test-patch</id>
       <activation>
         <activeByDefault>false</activeByDefault>

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
Thu May 23 20:41:53 2013
@@ -83,7 +83,7 @@ public class TestStreamingTaskLog {
    *  (b) hadoop.tasklog.totalLogFileSize
    * for the children of java tasks in streaming jobs.
    */
-  @Test
+  @Test (timeout = 30000)
   public void testStreamingTaskLogWithHadoopCmd() {
     try {
       final int numSlaves = 1;
@@ -124,8 +124,8 @@ public class TestStreamingTaskLog {
               "echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes());
     in.close();
     
-    Shell.execCommand(new String[]{"chmod", "+x",
-                                   scriptFile.getAbsolutePath()});
+    Shell.execCommand(Shell.getSetPermissionCommand("+x", false,
+        scriptFile.getAbsolutePath()));
     return scriptFile;
   }
   

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java?rev=1485845&r1=1485844&r2=1485845&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
Thu May 23 20:41:53 2013
@@ -53,7 +53,7 @@ public class TestSymLink
   String cacheString = "This is just the cache string";
   StreamJob job;
 
-  @Test
+  @Test (timeout = 60000)
   public void testSymLink() throws Exception
   {
     boolean mayExit = false;



Mime
View raw message