hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject svn commit: r1351818 - in /hadoop/common/trunk: hadoop-assemblies/src/main/resources/assemblies/ hadoop-tools/ hadoop-tools/hadoop-pipes/ hadoop-tools/hadoop-pipes/src/ hadoop-tools/hadoop-pipes/src/main/ hadoop-tools/hadoop-pipes/src/main/native/ hado...
Date Tue, 19 Jun 2012 19:40:45 GMT
Author: bobby
Date: Tue Jun 19 19:40:44 2012
New Revision: 1351818

URL: http://svn.apache.org/viewvc?rev=1351818&view=rev
Log:
MAPREDUCE-4267. mavenize pipes (tgraves via bobby)

Added:
    hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/pom.xml
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/
      - copied from r1351806, hadoop/common/trunk/hadoop-mapreduce-project/src/examples/pipes/
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/pipes/
      - copied from r1351806, hadoop/common/trunk/hadoop-mapreduce-project/src/c++/pipes/
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/utils/
      - copied from r1351806, hadoop/common/trunk/hadoop-mapreduce-project/src/c++/utils/
Removed:
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/.autom4te.cfg
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/Makefile.am
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/configure.ac
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/pipes/.autom4te.cfg
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/pipes/Makefile.am
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/pipes/configure.ac
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/utils/.autom4te.cfg
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/utils/Makefile.am
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/utils/configure.ac
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/utils/m4/hadoop_utils.m4
Modified:
    hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt
    hadoop/common/trunk/hadoop-tools/hadoop-tools-dist/pom.xml
    hadoop/common/trunk/hadoop-tools/pom.xml

Added: hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml?rev=1351818&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml (added)
+++ hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml Tue
Jun 19 19:40:44 2012
@@ -0,0 +1,67 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0
http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+  <id>hadoop-tools</id>
+  <formats>
+    <format>dir</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
+      <includes>
+        <include>*.hh</include>
+      </includes>
+      <outputDirectory>/include</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-pipes/src/main/native/utils/api/hadoop</directory>
+      <includes>
+        <include>*.hh</include>
+      </includes>
+      <outputDirectory>/include</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-pipes/target/native</directory>
+      <includes>
+        <include>*.a</include>
+      </includes>
+      <outputDirectory>lib/native</outputDirectory>
+    </fileSet>
+  </fileSets>
+  <dependencySets>
+    <dependencySet>
+      <outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
+      <unpack>false</unpack>
+      <scope>runtime</scope>
+      <useProjectArtifact>false</useProjectArtifact>
+      <!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
+      <excludes>
+        <exclude>org.apache.hadoop:hadoop-common</exclude>
+        <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
+        <exclude>org.apache.hadoop:hadoop-mapreduce</exclude>
+        <!-- pipes is native stuff, this just keeps pom from being package-->
+        <exclude>org.apache.hadoop:hadoop-pipes</exclude>
+        <!-- use slf4j from common to avoid multiple binding warnings -->
+        <exclude>org.slf4j:slf4j-api</exclude>
+        <exclude>org.slf4j:slf4j-log4j12</exclude>
+      </excludes>
+    </dependencySet>
+  </dependencySets>
+</assembly>

Added: hadoop/common/trunk/hadoop-tools/hadoop-pipes/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-pipes/pom.xml?rev=1351818&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-pipes/pom.xml (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-pipes/pom.xml Tue Jun 19 19:40:44 2012
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <groupId>org.apache.hadoop</groupId>
+  <artifactId>hadoop-pipes</artifactId>
+  <version>3.0.0-SNAPSHOT</version>
+  <description>Apache Hadoop Pipes</description>
+  <name>Apache Hadoop Pipes</name>
+  <packaging>pom</packaging>
+
+  <properties>
+    <hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir>
+  </properties>
+
+  <profiles>
+    <profile>
+      <id>native</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>make</id>
+                <phase>compile</phase>
+                <goals><goal>run</goal></goals>
+                <configuration>
+                  <target>
+                    <mkdir dir="${project.build.directory}/native"/>
+                    <exec executable="cmake" dir="${project.build.directory}/native" 
+                        failonerror="true">
+                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+                    </exec>
+                    <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
+                      <arg line="VERBOSE=1"/>
+                    </exec>
+                  </target>
+                </configuration>
+              </execution>
+              <!-- TODO wire here native testcases
+              <execution>
+                <id>test</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>test</goal>
+                </goals>
+                <configuration>
+                  <destDir>${project.build.directory}/native/target</destDir>
+                </configuration>
+              </execution>
+              -->
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles> 
+
+<!--
+  <build>
+    <plugins>
+     <plugin>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>compile</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <mkdir dir="${basedir}/../target/native"/>
+                <copy toDir="${basedir}/../target/native">
+                  <fileset dir="${basedir}/src/main/native"/>
+                </copy>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+-->
+</project>

Added: hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/CMakeLists.txt?rev=1351818&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/CMakeLists.txt (added)
+++ hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/CMakeLists.txt Tue Jun 19 19:40:44 2012
@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+ 
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
+find_package(OpenSSL REQUIRED)
+
+set(CMAKE_BUILD_TYPE, Release)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
+
+if (JVM_ARCH_DATA_MODEL EQUAL 32)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
+    set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
+    if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
+        set(CMAKE_SYSTEM_PROCESSOR "i686")
+    endif ()
+endif (JVM_ARCH_DATA_MODEL EQUAL 32)
+
+function(output_directory TGT DIR)
+    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+        RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
+   SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+        ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
+    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
+endfunction(output_directory TGT DIR)
+
+include_directories(
+    main/native/utils/api
+    main/native/pipes/api
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${OPENSSL_INCLUDE_DIR}
+)
+
+# Example programs
+add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
+target_link_libraries(wordcount-simple hadooppipes hadooputils)
+
+add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
+target_link_libraries(wordcount-part hadooppipes hadooputils)
+
+add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
+target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
+
+add_executable(pipes-sort main/native/examples/impl/sort.cc)
+target_link_libraries(pipes-sort hadooppipes hadooputils)
+
+install(TARGETS wordcount-simple wordcount-part wordcount-nopipe pipes-sort 
+    RUNTIME DESTINATION bin
+)
+
+add_library(hadooputils STATIC
+    main/native/utils/impl/StringUtils.cc
+    main/native/utils/impl/SerialUtils.cc
+)
+
+install(FILES
+    main/native/utils/api/hadoop/SerialUtils.hh
+    main/native/utils/api/hadoop/StringUtils.hh
+    DESTINATION api/hadoop
+    COMPONENT headers
+)
+install(TARGETS hadooputils DESTINATION lib)
+
+add_library(hadooppipes STATIC
+    main/native/pipes/impl/HadoopPipes.cc
+)
+target_link_libraries(hadooppipes
+    ${JAVA_JVM_LIBRARY}
+    ${OPENSSL_LIBRARIES}
+    pthread
+)
+
+install(FILES
+    main/native/pipes/api/hadoop/Pipes.hh
+    main/native/pipes/api/hadoop/TemplateFactory.hh
+    DESTINATION api/hadoop
+    COMPONENT headers
+)
+install(TARGETS hadooppipes DESTINATION lib)

Modified: hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt?rev=1351818&r1=1351806&r2=1351818&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt (original)
+++ hadoop/common/trunk/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt Tue
Jun 19 19:40:44 2012
@@ -1,16 +1,16 @@
 To run the examples, first compile them:
 
-% ant -Dcompile.c++=yes examples
+% mvn install 
 
 and then copy the binaries to dfs:
 
-% bin/hadoop fs -put build/c++-examples/Linux-i386-32/bin /examples/bin
+% hadoop fs -put target/native/wordcount-simple /examples/bin/
 
 create an input directory with text files:
 
-% bin/hadoop fs -put my-data in-dir
+% hadoop fs -put my-data in-dir
 
 and run the word count example:
 
-% bin/hadoop pipes -conf src/examples/pipes/conf/word.xml \
+% hadoop pipes -conf src/main/native/examples/conf/word.xml \
                    -input in-dir -output out-dir

Modified: hadoop/common/trunk/hadoop-tools/hadoop-tools-dist/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-tools-dist/pom.xml?rev=1351818&r1=1351817&r2=1351818&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-tools/hadoop-tools-dist/pom.xml (original)
+++ hadoop/common/trunk/hadoop-tools/hadoop-tools-dist/pom.xml Tue Jun 19 19:40:44 2012
@@ -70,6 +70,13 @@
       <artifactId>hadoop-gridmix</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-pipes</artifactId>
+      <scope>compile</scope>
+      <type>pom</type>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -92,4 +99,44 @@
     </plugins>
   </build>
 
+  <profiles>
+    <profile>
+      <id>dist</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <dependencies>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-assemblies</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+            </dependencies>
+            <executions>
+              <execution>
+                <id>dist</id>
+                <phase>prepare-package</phase>
+                <goals>
+                  <goal>single</goal>
+                </goals>
+                <configuration>
+                  <appendAssemblyId>false</appendAssemblyId>
+                  <attach>false</attach>
+                  <finalName>${project.artifactId}-${project.version}</finalName>
+                  <descriptorRefs>
+                    <descriptorRef>hadoop-tools</descriptorRef>
+                  </descriptorRefs>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>

Modified: hadoop/common/trunk/hadoop-tools/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/pom.xml?rev=1351818&r1=1351817&r2=1351818&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-tools/pom.xml (original)
+++ hadoop/common/trunk/hadoop-tools/pom.xml Tue Jun 19 19:40:44 2012
@@ -39,6 +39,7 @@
     <module>hadoop-datajoin</module>
     <module>hadoop-tools-dist</module>
     <module>hadoop-extras</module>
+    <module>hadoop-pipes</module>
   </modules>
 
   <build>



Mime
View raw message