falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srik...@apache.org
Subject [3/4] git commit: Revert "FALCON-88 Add embedded hive and webhcat for integration tests. Contributed by Venkatesh Seetharam"
Date Mon, 02 Sep 2013 03:15:46 GMT
Revert "FALCON-88 Add embedded hive and webhcat for integration tests. Contributed by Venkatesh
Seetharam"

This reverts commit 19bcf9994125bba86bf5ed79ef3d44c5bf0a2f8a.


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/f1db98b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/f1db98b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/f1db98b3

Branch: refs/heads/master
Commit: f1db98b378a36b749206682b3bc3900e0cad7716
Parents: 7b12881
Author: srikanth.sundarrajan <srikanth.sundarrajan@inmobi.com>
Authored: Mon Sep 2 07:56:10 2013 +0530
Committer: srikanth.sundarrajan <srikanth.sundarrajan@inmobi.com>
Committed: Mon Sep 2 07:56:10 2013 +0530

----------------------------------------------------------------------
 hadoop-webapp/pom.xml                           |  21 ---
 .../falcon/listener/HadoopStartupListener.java  | 140 ++++++-------------
 hadoop-webapp/src/main/resources/hive-site.xml  |  27 ----
 .../src/main/resources/webhcat-site.xml         |  29 ----
 pom.xml                                         |  30 +---
 webapp/pom.xml                                  |   1 -
 6 files changed, 44 insertions(+), 204 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f1db98b3/hadoop-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/pom.xml b/hadoop-webapp/pom.xml
index c8c51d6..312a71b 100644
--- a/hadoop-webapp/pom.xml
+++ b/hadoop-webapp/pom.xml
@@ -91,27 +91,6 @@
             <groupId>org.apache.activemq</groupId>
             <artifactId>activemq-core</artifactId>
         </dependency>
-
-        <dependency>
-            <groupId>org.apache.hive</groupId>
-            <artifactId>hive-metastore</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hcatalog</groupId>
-            <artifactId>webhcat</artifactId>
-        </dependency>
-
-        <!-- Hive Metastore and WebHcat fails with out these dependencies -->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-        </dependency>
     </dependencies>
 
     <build>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f1db98b3/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
b/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
index 570ba02..5114a4b 100644
--- a/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
+++ b/hadoop-webapp/src/main/java/org/apache/falcon/listener/HadoopStartupListener.java
@@ -19,8 +19,6 @@
 package org.apache.falcon.listener;
 
 import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
 
 import javax.servlet.ServletContextEvent;
 import javax.servlet.ServletContextListener;
@@ -30,9 +28,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hcatalog.templeton.AppConfig;
 import org.apache.log4j.Logger;
 
 /**
@@ -52,18 +48,54 @@ public class HadoopStartupListener implements ServletContextListener {
             final String[] emptyArgs = {};
             String hadoopProfle = System.getProperty("hadoop.profile", "1");
             if (hadoopProfle.equals("1")) {
-                startHadoop1Services(conf, emptyArgs);
+                NameNode.createNameNode(emptyArgs, conf);
+                DataNode.createDataNode(emptyArgs, conf);
+                JobConf jobConf = new JobConf(conf);
+                /**
+                 * Reflection code:
+                 * JobTracker jt = JobTracker.startTracker(jobConf);
+                 * jt.offerService();
+                 * TaskTracker tt = new TaskTracker(jobConf);
+                 * tt.run();
+                 */
+                Object jt = Class.forName("org.apache.hadoop.mapred.JobTracker")
+                                .getMethod("startTracker", JobConf.class).invoke(null, jobConf);
+                startService(jt, "offerService");
+                Object tt = Class.forName("org.apache.hadoop.mapred.TaskTracker")
+                                .getConstructor(JobConf.class).newInstance(jobConf);
+                startService(tt, "run");
             } else if (hadoopProfle.equals("2")) {
-                startHadoop2Services(conf, emptyArgs);
+                /**
+                 * Reflection code:
+                 * DefaultMetricsSystem.setMiniClusterMode(true);
+                 * ResourceManager resourceManager = new ResourceManager(new MemStore());
+                 * YarnConfiguration yarnConf = new YarnConfiguration(conf);
+                 * resourceManager.init(yarnConf);
+                 * resourceManager.start();
+                 * NodeManager nodeManager = new NodeManager();
+                 * nodeManager.init(yarnConf);
+                 * nodeManager.start();
+                 */
+                Class.forName("org.apache.hadoop.metrics2.lib.DefaultMetricsSystem")
+                                .getMethod("setMiniClusterMode", boolean.class).invoke(null,
true);
+                NameNode.createNameNode(emptyArgs, conf);
+                DataNode.createDataNode(emptyArgs, conf);
+
+                Object memStore = instance("org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore");
+                Object resourceManager = Class.forName("org.apache.hadoop.yarn.server.resourcemanager.ResourceManager")
+                        .getConstructor(Class.forName("org.apache.hadoop.yarn.server.resourcemanager.recovery.Store"))
+                        .newInstance(memStore);
+                Object yarnConf = Class.forName("org.apache.hadoop.yarn.conf.YarnConfiguration")
+                        .getConstructor(Configuration.class).newInstance(conf);
+                invoke(resourceManager, "init", Configuration.class, yarnConf);
+                startService(resourceManager, "start");
+                Object nodeManager = instance("org.apache.hadoop.yarn.server.nodemanager.NodeManager");
+                invoke(nodeManager, "init", Configuration.class, yarnConf);
+                startService(nodeManager, "start");
             } else {
                 throw new RuntimeException("Unhandled hadoop profile " + hadoopProfle);
             }
-
             startBroker();
-
-            startHiveMetaStore();
-            startHiveWebMetaStore();
-
         } catch (Exception e) {
             e.printStackTrace();
             LOG.error("Unable to start hadoop cluster", e);
@@ -71,59 +103,6 @@ public class HadoopStartupListener implements ServletContextListener {
         }
     }
 
-    private void startHadoop1Services(Configuration conf, String[] emptyArgs)
-        throws IOException, IllegalAccessException, InvocationTargetException,
-               NoSuchMethodException, ClassNotFoundException, InstantiationException {
-
-        NameNode.createNameNode(emptyArgs, conf);
-        DataNode.createDataNode(emptyArgs, conf);
-
-        JobConf jobConf = new JobConf(conf);
-        // JobTracker jt = JobTracker.startTracker(jobConf);
-        // jt.offerService();
-        // TaskTracker tt = new TaskTracker(jobConf);
-        // tt.run();
-
-        Object jt = Class.forName("org.apache.hadoop.mapred.JobTracker")
-                        .getMethod("startTracker", JobConf.class).invoke(null, jobConf);
-        startService(jt, "offerService");
-
-        Object tt = Class.forName("org.apache.hadoop.mapred.TaskTracker")
-                        .getConstructor(JobConf.class).newInstance(jobConf);
-        startService(tt, "run");
-    }
-
-    private void startHadoop2Services(Configuration conf, String[] emptyArgs) throws Exception
{
-
-        // DefaultMetricsSystem.setMiniClusterMode(true);
-        // ResourceManager resourceManager = new ResourceManager(new MemStore());
-        // YarnConfiguration yarnConf = new YarnConfiguration(conf);
-        // resourceManager.init(yarnConf);
-        // resourceManager.start();
-        // NodeManager nodeManager = new NodeManager();
-        // nodeManager.init(yarnConf);
-        // nodeManager.start();
-
-        Class.forName("org.apache.hadoop.metrics2.lib.DefaultMetricsSystem")
-                        .getMethod("setMiniClusterMode", boolean.class).invoke(null, true);
-
-        NameNode.createNameNode(emptyArgs, conf);
-        DataNode.createDataNode(emptyArgs, conf);
-
-        Object memStore = instance("org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore");
-        Object resourceManager = Class.forName("org.apache.hadoop.yarn.server.resourcemanager.ResourceManager")
-                .getConstructor(Class.forName("org.apache.hadoop.yarn.server.resourcemanager.recovery.Store"))
-                .newInstance(memStore);
-        Object yarnConf = Class.forName("org.apache.hadoop.yarn.conf.YarnConfiguration")
-                .getConstructor(Configuration.class).newInstance(conf);
-        invoke(resourceManager, "init", Configuration.class, yarnConf);
-        startService(resourceManager, "start");
-
-        Object nodeManager = instance("org.apache.hadoop.yarn.server.nodemanager.NodeManager");
-        invoke(nodeManager, "init", Configuration.class, yarnConf);
-        startService(nodeManager, "start");
-    }
-
     private void startBroker() throws Exception {
         broker = new BrokerService();
         broker.setUseJmx(false);
@@ -133,39 +112,6 @@ public class HadoopStartupListener implements ServletContextListener
{
         broker.start();
     }
 
-    public static final String META_STORE_PORT = "49083";
-    private void startHiveMetaStore() {
-        try {
-            new Thread(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        String[] args = new String[]{
-                            "-v",
-                            "-p", META_STORE_PORT,
-                        };
-
-                        HiveMetaStore.main(args);
-                    } catch (Throwable t) {
-                        throw new RuntimeException(t);
-                    }
-                }
-            }).start();
-        } catch (Exception e) {
-            throw new RuntimeException("Unable to start hive metastore server.", e);
-        }
-    }
-
-    public static final String WEB_HCAT_PORT = "48080";
-    private void startHiveWebMetaStore() {
-        String[] args = new String[]{
-            "-D" + AppConfig.PORT + "=" + WEB_HCAT_PORT,
-            "-D" + AppConfig.HADOOP_CONF_DIR + "=./target/webapps/hadoop/conf",
-        };
-
-        org.apache.hcatalog.templeton.Main.main(args);
-    }
-
     private Object instance(String clsName) throws Exception {
         return Class.forName(clsName).newInstance();
     }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f1db98b3/hadoop-webapp/src/main/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/hive-site.xml b/hadoop-webapp/src/main/resources/hive-site.xml
deleted file mode 100644
index 89d739b..0000000
--- a/hadoop-webapp/src/main/resources/hive-site.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<configuration>
-    <!-- Forcing the creation of the db dir under target -->
-    <property>
-        <name>javax.jdo.option.ConnectionURL</name>
-        <value>jdbc:derby:;databaseName=./target/metastore_db;create=true</value>
-    </property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f1db98b3/hadoop-webapp/src/main/resources/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-webapp/src/main/resources/webhcat-site.xml b/hadoop-webapp/src/main/resources/webhcat-site.xml
deleted file mode 100644
index 6d0772c..0000000
--- a/hadoop-webapp/src/main/resources/webhcat-site.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<configuration>
-
-    <property>
-        <name>templeton.hadoop.conf.dir</name>
-        <value>${env.HADOOP_CONF_DIR}</value>
-        <description>The path to the Hadoop configuration.</description>
-    </property>
-
-</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f1db98b3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7498a69..d211e21 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,11 +98,10 @@
 
         <hadoop.profile>1</hadoop.profile>
         <hadoop.version>1.1.2</hadoop.version>
-        <slf4j.version>1.6.1</slf4j.version>
+        <slf4j.version>1.2</slf4j.version>
         <oozie.version>3.2.2</oozie.version>
         <activemq.version>5.4.3</activemq.version>
         <hadoop-distcp.version>0.3</hadoop-distcp.version>
-        <hive.version>0.11.0</hive.version>
         <jetty.version>6.1.26</jetty.version>
         <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo>
         <skipCheck>false</skipCheck>
@@ -704,30 +703,6 @@
             </dependency>
 
             <dependency>
-                <groupId>org.apache.hive</groupId>
-                <artifactId>hive-metastore</artifactId>
-                <version>${hive.version}</version>
-                <exclusions>
-                    <exclusion> <!--Oozie already imports this-->
-                        <groupId>org.apache.derby</groupId>
-                        <artifactId>derby</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-
-            <dependency>
-                <groupId>org.apache.hcatalog</groupId>
-                <artifactId>webhcat</artifactId>
-                <version>${hive.version}</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>xerces</groupId>
-                        <artifactId>xercesImpl</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-
-            <dependency>
                 <groupId>net.sourceforge.findbugs</groupId>
                 <artifactId>annotations</artifactId>
                 <version>1.3.2</version>
@@ -906,9 +881,6 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <configuration>
-                    <skip>${skipCheck}</skip>
-                </configuration>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/f1db98b3/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index fc7f9e6..ffeeb46 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -174,7 +174,6 @@
                             <dependencySourceIncludes>
                                 <dependencySourceInclude>org.apache.falcon:*</dependencySourceInclude>
                             </dependencySourceIncludes>
-                            <skip>${skipCheck}</skip>
                         </configuration>
                     </execution>
                 </executions>


Mime
View raw message