hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1190616 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: INSTALL assembly/all.xml hadoop-yarn/pom.xml pom.xml
Date Fri, 28 Oct 2011 21:57:39 GMT
Author: tucu
Date: Fri Oct 28 21:57:38 2011
New Revision: 1190616

URL: http://svn.apache.org/viewvc?rev=1190616&view=rev
Log:
Merge -r 1182150:1182151 from trunk to branch. FIXES: HADOOP-7642

Modified:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/INSTALL
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/assembly/all.xml
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/pom.xml
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/pom.xml

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/INSTALL
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/INSTALL?rev=1190616&r1=1190615&r2=1190616&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/INSTALL (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/INSTALL Fri Oct 28 21:57:38
2011
@@ -2,49 +2,31 @@ To compile  Hadoop Mapreduce next follow
 
 Step 1) Install dependencies for yarn
 
-See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce/hadoop-yarn/README
+See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce-porject/hadoop-yarn/README
 Make sure protbuf library is in your library path or set: export LD_LIBRARY_PATH=/usr/local/lib
 
 Step 2) Checkout
 
 svn checkout http://svn.apache.org/repos/asf/hadoop/common/trunk
 
-Step 3) Build common
+Step 3) Build
 
-Go to common directory - choose your regular common build command
-Example: mvn clean install package -Pbintar -DskipTests
+Go to common directory - choose your regular common build command. For example:
 
-Step 4) Build HDFS 
-
-Go to hdfs directory
-ant veryclean mvn-install -Dresolvers=internal 
-
-Step 5) Build yarn and mapreduce
-
-Go to mapreduce directory
 export MAVEN_OPTS=-Xmx512m
+mvn clean package -Pdist -Dtar -DskipTests -Pnative
 
-mvn clean install assembly:assembly -DskipTests
-
-Copy in build.properties if appropriate - make sure eclipse.home not set
-ant veryclean tar -Dresolvers=internal 
+You can omit -Pnative it you don't want to build native packages.
 
-You will see a tarball in
-ls target/hadoop-mapreduce-0.23.0-SNAPSHOT-all.tar.gz  
+Step 4) Untar the tarball from hadoop-dist/target/ into a clean and different
+directory, say YARN_HOME.
 
-Step 6) Untar the tarball in a clean and different directory.
-say YARN_HOME. 
-
-Make sure you aren't picking up avro-1.3.2.jar, remove:
-  $HADOOP_COMMON_HOME/share/hadoop/common/lib/avro-1.3.2.jar
-  $YARN_HOME/lib/avro-1.3.2.jar
-
-Step 7)
-Install hdfs/common and start hdfs
+Step 5)
+Start hdfs
 
 To run Hadoop Mapreduce next applications: 
 
-Step 8) export the following variables to where you have things installed:
+Step 6) export the following variables to where you have things installed:
 You probably want to export these in hadoop-env.sh and yarn-env.sh also.
 
 export HADOOP_MAPRED_HOME=<mapred loc>
@@ -54,7 +36,7 @@ export YARN_HOME=directory where you unt
 export HADOOP_CONF_DIR=<conf loc>
 export YARN_CONF_DIR=$HADOOP_CONF_DIR
 
-Step 9) Setup config: for running mapreduce applications, which now are in user land, you
need to setup nodemanager with the following configuration in your yarn-site.xml before you
start the nodemanager.
+Step 7) Setup config: for running mapreduce applications, which now are in user land, you
need to setup nodemanager with the following configuration in your yarn-site.xml before you
start the nodemanager.
     <property>
       <name>yarn.nodemanager.aux-services</name>
       <value>mapreduce.shuffle</value>
@@ -65,31 +47,21 @@ Step 9) Setup config: for running mapred
       <value>org.apache.hadoop.mapred.ShuffleHandler</value>
     </property>
 
-Step 10) Modify mapred-site.xml to use yarn framework
+Step 8) Modify mapred-site.xml to use yarn framework
     <property>    
       <name> mapreduce.framework.name</name>
       <value>yarn</value>  
     </property>
 
-Step 11) Create the following symlinks in $HADOOP_COMMON_HOME/share/hadoop/common/lib
-
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar .	
-ln -s $YARN_HOME/modules/hadoop-yarn-api-0.23.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-common-0.23.0-SNAPSHOT.jar .	
-ln -s $YARN_HOME/modules/hadoop-yarn-common-0.23.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-core-0.23.0-SNAPSHOT.jar .	
-ln -s $YARN_HOME/modules/hadoop-yarn-server-common-0.23.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar .
-
-Step 12) cd $YARN_HOME
+Step 9) cd $YARN_HOME
 
-Step 13) bin/yarn-daemon.sh start resourcemanager
+Step 10) bin/yarn-daemon.sh start resourcemanager
 
-Step 14) bin/yarn-daemon.sh start nodemanager
+Step 11) bin/yarn-daemon.sh start nodemanager
 
-Step 15) bin/yarn-daemon.sh start historyserver
+Step 12) bin/yarn-daemon.sh start historyserver
 
-Step 16) You are all set, an example on how to run a mapreduce job is:
+Step 13) You are all set, an example on how to run a mapreduce job is:
 cd $HADOOP_MAPRED_HOME
 ant examples -Dresolvers=internal 
 $HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-0.23.0-SNAPSHOT.jar
randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory
-Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912
-libjars $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar output 

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/assembly/all.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/assembly/all.xml?rev=1190616&r1=1190615&r2=1190616&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/assembly/all.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/assembly/all.xml Fri Oct 28
21:57:38 2011
@@ -1,101 +0,0 @@
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0
http://maven.apache.org/xsd/assembly-1.1.0.xsd">
-  <id>all</id>
-  <formats>
-    <format>tar.gz</format>
-  </formats>
-  <includeBaseDirectory>true</includeBaseDirectory>
-  <!-- TODO: this layout is wrong. We need module specific bin files in module specific
dirs -->
-  <fileSets>
-    <fileSet>
-      <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/c/container-executor</directory>
-      <outputDirectory>bin</outputDirectory>
-      <includes>
-        <include>container-executor</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>hadoop-yarn/bin</directory>
-      <outputDirectory>bin</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>bin</directory>
-      <outputDirectory>bin</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>hadoop-yarn/conf</directory>
-      <outputDirectory>conf</outputDirectory>
-      <includes>
-        <include>**/*</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <outputDirectory>sources</outputDirectory>
-      <excludes>
-        <exclude>**/*.jar</exclude>
-        <exclude>**/target/**</exclude>
-        <!-- scripts to include later for setting fileMode -->
-        <exclude>**/bin/*</exclude>
-        <exclude>**/scripts/*</exclude>
-        <!-- images that we don't need (and cause problems for our tools) -->
-        <exclude>**/dt-*/images/**</exclude>
-        <!-- until the code that does this is fixed -->
-        <exclude>**/file:/**</exclude>
-        <exclude>**/SecurityAuth.audit*</exclude>
-      </excludes>
-      <includes>
-        <include>assembly/**</include>
-        <include>pom.xml</include>
-        <include>build*.xml</include>
-        <include>ivy.xml</include>
-        <include>ivy/**</include>
-        <include>INSTALL</include>
-        <include>LICENSE.txt</include>
-        <include>mr-client/**</include>
-        <include>hadoop-yarn/**</include>
-        <include>src/**</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <outputDirectory>sources</outputDirectory>
-      <includes>
-        <include>**/bin/*</include>
-        <include>**/scripts/*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-  </fileSets>
-  <moduleSets>
-    <moduleSet>
-      <excludes>
-        <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
-      </excludes>
-      <binaries>
-        <outputDirectory>modules</outputDirectory>
-        <includeDependencies>false</includeDependencies>
-        <unpack>false</unpack>
-      </binaries>
-    </moduleSet>
-  </moduleSets>
-  <dependencySets>
-    <dependencySet>
-      <useProjectArtifact>false</useProjectArtifact>
-      <outputDirectory>/lib</outputDirectory>
-      <!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
-      <excludes>
-        <exclude>org.apache.hadoop:hadoop-common</exclude>
-        <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
-      </excludes>
-    </dependencySet>
-  </dependencySets>
-</assembly>

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/pom.xml?rev=1190616&r1=1190615&r2=1190616&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/pom.xml Fri Oct
28 21:57:38 2011
@@ -25,6 +25,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <test.logs>true</test.logs>
     <test.timeout>600000</test.timeout>
+    <hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
     <hadoop-common.version>0.23.0-SNAPSHOT</hadoop-common.version>
     <hadoop-hdfs.version>0.23.0-SNAPSHOT</hadoop-hdfs.version>
     <yarn.version>0.23.0-SNAPSHOT</yarn.version>
@@ -329,13 +330,33 @@
     </pluginManagement>
     <plugins>
       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-assembly-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-assemblies</artifactId>
+            <version>${hadoop.assemblies.version}</version>
+          </dependency>
+        </dependencies>
         <configuration>
           <tarLongFileMode>gnu</tarLongFileMode>
-          <descriptors>
-            <descriptor>assembly/all.xml</descriptor>
-          </descriptors>
+          <appendAssemblyId>false</appendAssemblyId>
+          <attach>false</attach>
+          <finalName>${project.artifactId}-${project.version}</finalName>
+          <descriptorRefs>
+            <descriptorRef>hadoop-mapreduce-dist</descriptorRef>
+          </descriptorRefs>
         </configuration>
+        <executions>
+          <execution>
+            <id>tar</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
       </plugin>
       <plugin>
         <artifactId>maven-antrun-plugin</artifactId>

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/pom.xml?rev=1190616&r1=1190615&r2=1190616&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/pom.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/pom.xml Fri Oct 28 21:57:38
2011
@@ -26,6 +26,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <test.logs>true</test.logs>
     <test.timeout>600000</test.timeout>
+    <hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
     <hadoop-common.version>0.23.0-SNAPSHOT</hadoop-common.version>
     <hadoop-hdfs.version>0.23.0-SNAPSHOT</hadoop-hdfs.version>
     <hadoop-mapreduce.version>0.23.0-SNAPSHOT</hadoop-mapreduce.version>
@@ -325,13 +326,33 @@
         </executions>
       </plugin>
       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-assembly-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-assemblies</artifactId>
+            <version>${hadoop.assemblies.version}</version>
+          </dependency>
+        </dependencies>
         <configuration>
           <tarLongFileMode>gnu</tarLongFileMode>
-          <descriptors>
-            <descriptor>assembly/all.xml</descriptor>
-          </descriptors>
+          <appendAssemblyId>false</appendAssemblyId>
+          <attach>false</attach>
+          <finalName>${project.artifactId}-${project.version}</finalName>
+          <descriptorRefs>
+            <descriptorRef>hadoop-mapreduce-dist</descriptorRef>
+          </descriptorRefs>
         </configuration>
+        <executions>
+          <execution>
+            <id>tar</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
       </plugin>
       <plugin>
         <groupId>com.atlassian.maven.plugins</groupId>



Mime
View raw message