falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject git commit: FALCON-63 Ability to ingest hadoop libs to falcon release package. Contributed by Suhas Vasu
Date Wed, 28 Aug 2013 11:27:06 GMT
Updated Branches:
  refs/heads/master 16972af86 -> 19ba0368e


FALCON-63 Ability to ingest hadoop libs to falcon release package. Contributed by Suhas Vasu


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/19ba0368
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/19ba0368
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/19ba0368

Branch: refs/heads/master
Commit: 19ba0368ea6f9bbb960781ca7dbafd93b084ad00
Parents: 16972af
Author: Shwetha GS <shwetha.gs@inmobi.com>
Authored: Wed Aug 28 17:01:48 2013 +0530
Committer: Shwetha GS <shwetha.gs@inmobi.com>
Committed: Wed Aug 28 17:01:48 2013 +0530

----------------------------------------------------------------------
 CHANGES.txt                            |  3 +
 hadoop-dependencies/pom.xml            | 87 +++++++++++++++++++++++++++++
 pom.xml                                |  1 +
 prism/pom.xml                          | 37 ++++++++++++
 src/bin/service-start.sh               | 19 +++++++
 src/main/assemblies/bin-package.xml    |  5 ++
 src/main/assemblies/prism-package.xml  |  5 ++
 src/main/assemblies/server-package.xml |  5 ++
 webapp/pom.xml                         | 35 ++++++++++++
 9 files changed, 197 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 12188bb..675da6c 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,9 @@ Trunk (Unreleased)
     via Srikanth Sundarrajan)
 
   IMPROVEMENTS
+    FALCON-63 Ability to ingest hadoop libs to falcon release package.
+    (Suhas Vasu via Shwetha GS)
+
     FALCON-83 Option to build distributed packages. (Shwetha GS via
     Srikanth Sundarrajan)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/hadoop-dependencies/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dependencies/pom.xml b/hadoop-dependencies/pom.xml
new file mode 100644
index 0000000..98f5a29
--- /dev/null
+++ b/hadoop-dependencies/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.falcon</groupId>
+        <artifactId>falcon-main</artifactId>
+        <version>0.4-incubating-SNAPSHOT</version>
+    </parent>
+    <artifactId>falcon-hadoop-dependencies</artifactId>
+    <description>Apache Falcon Hadoop Dependencies Module</description>
+    <name>Apache Falcon Hadoop Dependencies</name>
+
+    <profiles>
+        <profile>
+            <id>hadoop-1</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>hadoop.profile</name>
+                    <value>1</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-core</artifactId>
+                </dependency>
+            </dependencies>
+        </profile>
+
+        <profile>
+            <id>hadoop-2</id>
+            <activation>
+                <property>
+                    <name>hadoop.profile</name>
+                    <value>2</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-client</artifactId>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/dependency</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 53149b0..aeba853 100644
--- a/pom.xml
+++ b/pom.xml
@@ -147,6 +147,7 @@
         <module>metrics</module>
         <module>common</module>
         <module>test-util</module>
+        <module>hadoop-dependencies</module>
         <module>hadoop-webapp</module>
         <module>feed</module>
         <module>messaging</module>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/prism/pom.xml
----------------------------------------------------------------------
diff --git a/prism/pom.xml b/prism/pom.xml
index 55439c1..2d78ffd 100644
--- a/prism/pom.xml
+++ b/prism/pom.xml
@@ -173,4 +173,41 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>hadoop-1</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>hadoop.profile</name>
+                    <value>1</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-core</artifactId>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+        </profile>
+
+        <profile>
+            <id>hadoop-2</id>
+            <activation>
+                <property>
+                    <name>hadoop.profile</name>
+                    <value>2</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-client</artifactId>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/bin/service-start.sh
----------------------------------------------------------------------
diff --git a/src/bin/service-start.sh b/src/bin/service-start.sh
index a117d3f..5ebcbdf 100755
--- a/src/bin/service-start.sh
+++ b/src/bin/service-start.sh
@@ -56,6 +56,23 @@ for i in "${BASEDIR}/server/webapp/$APP_TYPE/WEB-INF/lib/"*.jar; do
   FALCONCPPATH="${FALCONCPPATH}:$i"
 done
 
+HADOOPDIR=`which hadoop`
+if [ "$HADOOPDIR" != "" ]; then
+  echo "Hadoop is installed, adding hadoop classpath to falcon classpath"
+  FALCONCPPATH="${FALCONCPPATH}:`hadoop classpath`"
+elif [ "$HADOOP_HOME" != "" ]; then
+  echo "Hadoop home is set, adding ${HADOOP_HOME}/lib/* into falcon classpath"
+  for i in "${HADOOP_HOME}/lib/"*.jar; do
+    FALCONCPPATH="${FALCONCPPATH}:$i"
+  done
+else
+  echo "Could not find installed hadoop and HADOOP_HOME is not set."
+  echo "Using the default jars bundled in ${BASEDIR}/hadooplibs/"
+  for i in "${BASEDIR}/hadooplibs/"*.jar; do
+    FALCONCPPATH="${FALCONCPPATH}:$i"
+  done
+fi
+
 if [ -z "$FALCON_CONF" ]; then
   CONF_PATH=${BASEDIR}/conf
 else
@@ -75,3 +92,5 @@ TIME=`date +%Y%m%d%H%M%s`
 nohup ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.Main -app ${BASEDIR}/server/webapp/*.war
$* 2> ${BASEDIR}/logs/$APP_TYPE.out.$TIME &
 echo $! > ${BASEDIR}/logs/$APP_TYPE.pid
 popd > /dev/null
+
+echo "Falcon started using hadoop version: " `${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH}
org.apache.hadoop.util.VersionInfo | head -1`

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/main/assemblies/bin-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/bin-package.xml b/src/main/assemblies/bin-package.xml
index 28d0449..92efad8 100644
--- a/src/main/assemblies/bin-package.xml
+++ b/src/main/assemblies/bin-package.xml
@@ -78,6 +78,11 @@
                 <include>README</include>
             </includes>
         </fileSet>
+
+        <fileSet>
+            <directory>hadoop-dependencies/target/dependency</directory>
+            <outputDirectory>hadooplibs</outputDirectory>
+        </fileSet>
     </fileSets>
 
     <files>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/main/assemblies/prism-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/prism-package.xml b/src/main/assemblies/prism-package.xml
index 943f387..434d8bb 100644
--- a/src/main/assemblies/prism-package.xml
+++ b/src/main/assemblies/prism-package.xml
@@ -79,6 +79,11 @@
                 <include>README</include>
             </includes>
         </fileSet>
+
+        <fileSet>
+            <directory>hadoop-dependencies/target/dependency</directory>
+            <outputDirectory>hadooplibs</outputDirectory>
+        </fileSet>
     </fileSets>
 
     <files>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/src/main/assemblies/server-package.xml
----------------------------------------------------------------------
diff --git a/src/main/assemblies/server-package.xml b/src/main/assemblies/server-package.xml
index c62218b..c92ad17 100644
--- a/src/main/assemblies/server-package.xml
+++ b/src/main/assemblies/server-package.xml
@@ -78,6 +78,11 @@
                 <include>README</include>
             </includes>
         </fileSet>
+
+        <fileSet>
+            <directory>hadoop-dependencies/target/dependency</directory>
+            <outputDirectory>hadooplibs</outputDirectory>
+        </fileSet>
     </fileSets>
 
     <files>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/19ba0368/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 2b3622b..ffeeb46 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -407,6 +407,41 @@
                 </plugins>
             </build>
         </profile>
+
+        <profile>
+            <id>hadoop-1</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>hadoop.profile</name>
+                    <value>1</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-core</artifactId>
+                    <scope>test</scope>
+                </dependency>
+            </dependencies>
+        </profile>
+
+        <profile>
+            <id>hadoop-2</id>
+            <activation>
+                <property>
+                    <name>hadoop.profile</name>
+                    <value>2</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-client</artifactId>
+                    <scope>test</scope>
+                </dependency>
+            </dependencies>
+        </profile>
     </profiles>
 
 </project>


Mime
View raw message