eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject [2/2] incubator-eagle git commit: EAGLE-477 eagle-data-process project clean up to contain only common processing eagle-data-process project clean up to contain only common processing
Date Fri, 19 Aug 2016 17:23:58 GMT
EAGLE-477 eagle-data-process project clean up to contain only common processing
eagle-data-process project clean up to contain only common processing

https://issues.apache.org/jira/browse/EAGLE-477
Author: @yonzhang <yonzhang2012@apache.org>

Closes: #366


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/b31bac50
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/b31bac50
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/b31bac50

Branch: refs/heads/develop
Commit: b31bac50b267fbd72b543eba65594c3a98bc67fc
Parents: d6ec142
Author: yonzhang <yonzhang2012@gmail.com>
Authored: Fri Aug 19 10:27:53 2016 -0700
Committer: yonzhang <yonzhang2012@gmail.com>
Committed: Fri Aug 19 10:27:53 2016 -0700

----------------------------------------------------------------------
 .../eagle-alert-service/pom.xml                 |   2 +-
 .../service/alert/PolicyValidateResource.java   |  64 ------
 .../src/main/resources/core-site.xml            |  24 --
 .../src/test/resources/log4j.properties         |  35 ---
 .../eagle-stream-process-api/pom.xml            | 145 ------------
 .../dataproc/impl/storm/StormSpoutProvider.java |  29 ---
 .../storm/hdfs/DataCollectionHDFSSpout.java     | 230 -------------------
 .../hdfs/HDFSSourcedStormSpoutProvider.java     |  58 -----
 .../impl/storm/kafka/KafkaSpoutProvider.java    | 130 -----------
 .../kafka/SpoutKafkaMessageDeserializer.java    |  23 --
 .../partition/CustomPartitionGrouping.java      |  51 ----
 .../impl/storm/zookeeper/ZKStateConfig.java     |  28 ---
 .../src/main/resources/application.conf         |  72 ------
 .../src/main/resources/log4j.properties         |  21 --
 .../eagle/datastream/utils/NameConstants.scala  |  24 --
 .../eagle-stream-process-base/pom.xml           | 128 -----------
 .../dataproc/core/EagleOutputCollector.java     |  27 ---
 .../eagle/dataproc/core/JsonSerDeserUtils.java  |  54 -----
 .../apache/eagle/dataproc/core/ValuesArray.java |  38 ---
 .../util/AbstractConfigOptionParser.java        |  68 ------
 .../eagle/dataproc/util/ConfigOptionParser.java |  67 ------
 .../eagle/partition/DataDistributionDao.java    |  28 ---
 .../eagle/partition/PartitionAlgorithm.java     |  29 ---
 .../eagle/partition/PartitionStrategy.java      |  27 ---
 .../eagle/partition/PartitionStrategyImpl.java  |  86 -------
 .../java/org/apache/eagle/partition/Weight.java |  30 ---
 .../org/apache/eagle/datastream/Collector.scala |  21 --
 eagle-core/eagle-data-process/pom.xml           | 136 ++++++++++-
 .../dataproc/impl/storm/StormSpoutProvider.java |  29 +++
 .../eagle/dataproc/impl/storm/ValuesArray.java  |  38 +++
 .../impl/storm/kafka/KafkaSpoutProvider.java    | 124 ++++++++++
 .../kafka/SpoutKafkaMessageDeserializer.java    |  23 ++
 .../partition/CustomPartitionGrouping.java      |  48 ++++
 .../storm/partition/DataDistributionDao.java    |  26 +++
 .../storm/partition/PartitionAlgorithm.java     |  26 +++
 .../impl/storm/partition/PartitionStrategy.java |  25 ++
 .../storm/partition/PartitionStrategyImpl.java  |  84 +++++++
 .../dataproc/impl/storm/partition/Weight.java   |  28 +++
 .../impl/storm/zookeeper/ZKStateConfig.java     |  28 +++
 .../src/main/resources/application.conf         |  72 ++++++
 .../src/main/resources/log4j.properties         |  21 ++
 .../eagle-policy/eagle-policy-base/pom.xml      |   2 +-
 .../apache/eagle/policy/common/Constants.java   |  79 -------
 .../eagle/policy/siddhi/AttributeType.java      |  30 ---
 .../src/main/resources/eagle.siddhiext          |  16 --
 eagle-gc/pom.xml                                |   2 +-
 .../gc/executor/GCMetricGeneratorBolt.java      |   4 +-
 eagle-jpm/eagle-hadoop-queue/pom.xml            |   4 +-
 .../hadoop/queue/HadoopQueueRunningMain.java    |   4 +-
 .../crawler/ClusterMetricsParseListener.java    |   2 +-
 .../queue/crawler/RunningAppParseListener.java  |   2 +-
 .../crawler/SchedulerInfoParseListener.java     |   2 +-
 eagle-jpm/eagle-jpm-mr-history/pom.xml          |  27 +--
 .../jpm/mr/history/common/JHFConfigManager.java |   2 +-
 .../crawler/DefaultJHFInputStreamCallback.java  |   7 +-
 .../history/crawler/EagleOutputCollector.java   |  29 +++
 .../JobHistorySpoutCollectorInterceptor.java    |   7 +-
 .../parser/JobEntityLifecycleAggregator.java    |   7 +-
 eagle-jpm/eagle-jpm-mr-running/pom.xml          |  19 +-
 .../running/config/MRRunningConfigManager.java  |   2 +-
 eagle-jpm/eagle-jpm-spark-running/pom.xml       |  17 +-
 .../spark/running/SparkRunningJobAppConfig.java |   2 +-
 eagle-security/eagle-metric-collection/pom.xml  |   9 +-
 .../kafka/EagleMetricCollectorApplication.java  |  10 +-
 .../metric/kafka/KafkaSourcedSpoutScheme.java   |  20 +-
 .../eagle/metric/kafka/NameConstants.java       |  25 ++
 .../kafka/TestDataDistributionDaoImpl.java      |   2 +-
 .../eagle/metric/kafka/TestGreedyPartition.java |   4 +-
 eagle-security/eagle-security-common/pom.xml    |   2 +-
 .../partition/DataDistributionDaoImpl.java      |   4 +-
 .../partition/GreedyPartitionAlgorithm.java     |   4 +-
 .../AbstractHdfsAuditLogApplication.java        |   6 +-
 .../eagle/security/auditlog/AttributeType.java  |  30 +++
 .../auditlog/HdfsUserCommandReassembler.java    |   3 +-
 .../hive/jobrunning/HiveJobFetchSpout.java      |   2 +-
 75 files changed, 842 insertions(+), 1792 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-alert-parent/eagle-alert-service/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-service/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert-service/pom.xml
index 81eddb1..d173c4a 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert-service/pom.xml
+++ b/eagle-core/eagle-alert-parent/eagle-alert-service/pom.xml
@@ -57,7 +57,7 @@
 		</dependency>
 		<dependency>
 			<groupId>org.apache.eagle</groupId>
-			<artifactId>eagle-stream-process-base</artifactId>
+			<artifactId>eagle-data-process</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 	</dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/PolicyValidateResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/PolicyValidateResource.java b/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/PolicyValidateResource.java
deleted file mode 100644
index e4c19d6..0000000
--- a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/PolicyValidateResource.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.service.alert;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.ServiceLoader;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.eagle.dataproc.core.JsonSerDeserUtils;
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import com.fasterxml.jackson.databind.Module;
-
-@Path("/policy/validate")
-public class PolicyValidateResource {
-		
-	public static Logger LOG = LoggerFactory.getLogger(PolicyValidateResource.class);
-	
-	@SuppressWarnings({"rawtypes"})
-	@POST
-	@Consumes(MediaType.APPLICATION_JSON)
-	@Produces(MediaType.APPLICATION_JSON)
-	public GenericServiceAPIResponseEntity validatePolicy(String policyToValidate) {
-        ServiceLoader<AlertPolicyValidateProvider> loader = ServiceLoader.load(AlertPolicyValidateProvider.class);
-        Iterator<AlertPolicyValidateProvider> iter = loader.iterator();
-        List<Module> modules = new ArrayList<Module>();
-        while(iter.hasNext()) {
-        	AlertPolicyValidateProvider factory = iter.next();
-            LOG.info("Supported policy type : " + factory.PolicyType());
-            modules.addAll(factory.BindingModules());
-        }
-        AlertPolicyValidateProvider policyValidate = null;
-        try {
-        	policyValidate = JsonSerDeserUtils.deserialize(policyToValidate, AlertPolicyValidateProvider.class, modules);    		
-        }
-        catch (Exception ex) {
-        	LOG.error("Fail consutructing AlertPolicyValidateProvider ", ex);
-        }
-        return policyValidate.validate();
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/resources/core-site.xml b/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/resources/core-site.xml
deleted file mode 100644
index 2e0c803..0000000
--- a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/resources/core-site.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-	<property>
-        <name>fs.defaultFS</name>
-        <value>hdfs://server.eagle.apache.org:8020</value>
-	</property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-alert-parent/eagle-alert-service/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-service/src/test/resources/log4j.properties b/eagle-core/eagle-alert-parent/eagle-alert-service/src/test/resources/log4j.properties
deleted file mode 100644
index 3499c46..0000000
--- a/eagle-core/eagle-alert-parent/eagle-alert-service/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, stdout
-
- eagle.log.dir=./logs
- eagle.log.file=eagle.log
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
-
-# Daily Rolling File Appender
- log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
- log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
- log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-## 30-day backup
-# log4j.appender.DRFA.MaxBackupIndex=30
- log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/pom.xml b/eagle-core/eagle-data-process/eagle-stream-process-api/pom.xml
deleted file mode 100644
index 5e3fc9e..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/pom.xml
+++ /dev/null
@@ -1,145 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.eagle</groupId>
-    <artifactId>eagle-data-process-parent</artifactId>
-    <version>0.5.0-incubating-SNAPSHOT</version>
-      <relativePath>../pom.xml</relativePath>
-  </parent>
-
-  <artifactId>eagle-stream-process-api</artifactId>
-  <name>eagle-stream-process-api</name>
-
-  <dependencies>
-      <dependency>
-        <groupId>org.apache.storm</groupId>
-        <artifactId>storm-core</artifactId>
-        <exclusions>
-            <exclusion>
-                <groupId>ch.qos.logback</groupId>
-                <artifactId>logback-classic</artifactId>
-            </exclusion>
-            <exclusion>
-                <groupId>log4j</groupId>
-                <artifactId>log4j</artifactId>
-            </exclusion>
-            <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>log4j-over-slf4j</artifactId>
-            </exclusion>
-        </exclusions>
-      </dependency>
-      <dependency>
-          <groupId>org.apache.storm</groupId>
-          <artifactId>storm-kafka</artifactId>
-          <exclusions>
-              <exclusion>
-                  <groupId>ch.qos.logback</groupId>
-                  <artifactId>logback-classic</artifactId>
-              </exclusion>
-              <exclusion>
-                  <groupId>log4j</groupId>
-                  <artifactId>log4j</artifactId>
-              </exclusion>
-              <exclusion>
-                  <groupId>org.slf4j</groupId>
-                  <artifactId>log4j-over-slf4j</artifactId>
-              </exclusion>
-          </exclusions>
-      </dependency>
-      <dependency>
-          <groupId>org.apache.eagle</groupId>
-          <artifactId>eagle-stream-process-base</artifactId>
-          <version>${project.version}</version>
-      </dependency>
-      <dependency>
-          <groupId>org.scala-lang</groupId>
-          <artifactId>scala-reflect</artifactId>
-          <exclusions>
-              <exclusion>
-                  <groupId>org.scala-lang</groupId>
-                  <artifactId>scala-libaray</artifactId>
-              </exclusion>
-          </exclusions>
-      </dependency>
-      <dependency>
-          <groupId>org.scalatest</groupId>
-          <artifactId>scalatest_${scala.version}</artifactId>
-          <scope>test</scope>
-      </dependency>
-
-	<dependency>
-		<groupId>org.apache.kafka</groupId>
-		<artifactId>kafka-clients</artifactId>
-		<version>${kafka-clients.version}</version>
-	</dependency>
-  </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.scala-tools</groupId>
-                <artifactId>maven-scala-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>scala-compile-first</id>
-                        <phase>process-resources</phase>
-                        <goals>
-                            <goal>add-source</goal>
-                            <goal>compile</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>scala-test-compile</id>
-                        <phase>process-test-resources</phase>
-                        <goals>
-                            <goal>testCompile</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <configuration>
-                    <skipTests>true</skipTests>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.scalatest</groupId>
-                <artifactId>scalatest-maven-plugin</artifactId>
-                <configuration>
-                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-                    <junitxml>.</junitxml>
-                    <filereports>TestSuite.txt</filereports>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>test</id>
-                        <goals>
-                            <goal>test</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
deleted file mode 100644
index ab90ad7..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/StormSpoutProvider.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.impl.storm;
-
-import backtype.storm.topology.base.BaseRichSpout;
-
-import com.typesafe.config.Config;
-
-/**
- * Normally storm spout is a special part of storm topology and it is implemented in underlying spout implementation
- * which can be retrieved from getSpout method.
- */
-public interface StormSpoutProvider {
-	public BaseRichSpout getSpout(Config context);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/DataCollectionHDFSSpout.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/DataCollectionHDFSSpout.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/DataCollectionHDFSSpout.java
deleted file mode 100644
index 2b00b92..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/DataCollectionHDFSSpout.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.impl.storm.hdfs;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.*;
-import java.util.zip.GZIPInputStream;
-
-import com.typesafe.config.Config;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import backtype.storm.spout.SpoutOutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.utils.Utils;
-
-import org.apache.eagle.dataproc.impl.storm.hdfs.HDFSSourcedStormSpoutProvider.HDFSSpout;
-
-public class DataCollectionHDFSSpout extends HDFSSpout{
-
-	private static final long serialVersionUID = 8775646842131298552L;
-	private Config configContext;
-	private TopologyContext _context; 
-	SpoutOutputCollector _collector;
-	private Map<String, Boolean> processFileMap = null; 
-	private static final Logger LOG = LoggerFactory.getLogger(DataCollectionHDFSSpout.class);
-	
-	public DataCollectionHDFSSpout(Config configContext){
-		this.configContext = configContext;
-		processFileMap = new HashMap<String, Boolean>();
-		LOG.info("DataCollectionHDFSSpout called");
-		
-	}
-	
-	public void copyFiles(){
-		LOG.info("Inside listFiles()");
-		Configuration conf = new Configuration(); 
-		// _____________ TO TEST THAT CORRECT HADOOP JARs ARE INCLUDED __________________
-		ClassLoader cl = ClassLoader.getSystemClassLoader();
-        URL[] urls = ((URLClassLoader)cl).getURLs();
-		if(LOG.isDebugEnabled()) {
-			for (URL url : urls) {
-				LOG.debug(url.getFile());
-			}
-		}
-		// _________________________________________
-        String hdfsConnectionStr = configContext.getString("dataSourceConfig.hdfsConnnection");
-        LOG.info("HDFS connection string: " + hdfsConnectionStr);
-       
-		String hdfsPath = configContext.getString("dataSourceConfig.hdfsPath");
-		LOG.info("HDFS path: " + hdfsPath);
-		 
-		String copyToPath = configContext.getString("dataSourceConfig.copyToPath");
-		LOG.info("copyToPath: " + copyToPath);
-		String srcPathStr = new String("hdfs://" + hdfsConnectionStr + hdfsPath);
-		Path srcPath = new Path(srcPathStr); 
-		LOG.info("listFiles called");
-		LOG.info("srcPath: " + srcPath);
-		try {
-			FileSystem fs = srcPath.getFileSystem(conf);
-			/*CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); 
-			CompressionCodec codec = codecFactory.getCodec(srcPath);
-			DataInputStream inputStream = new DataInputStream(codec.createInputStream(fs.open(srcPath)));
-			*/
-			
-			Path destPath = new Path(copyToPath);
-			LOG.info("Destination path: " + destPath);
-			fs.copyToLocalFile(srcPath, destPath);
-			LOG.info("Copy to local succeed");
-			fs.close();
-							
-		} catch (IOException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-		
-	}
-	
-	private List<String> getAllFiles(String root, int level){
-		
-		List<String> lists = new ArrayList<String>();
-		File rootFile = new File(root);
-		File[] tempList = rootFile.listFiles();
-		if(tempList == null)
-			return lists; 
-		
-		for(File temp:tempList){
-			if(temp.isDirectory())
-				lists.addAll(getAllFiles(temp.getAbsolutePath(), ++level));
-			else{
-				if(temp.getName().endsWith(".gz") || temp.getName().endsWith(".csv"))
-					lists.add(temp.getAbsolutePath());
-			}
-		}
-		return lists;
-			
-	}
-	
-	public List<String> listFiles(){
-		
-		String copyToPath = configContext.getString("dataSourceConfig.copyToPath");
-		LOG.info("Reading from: " + copyToPath);
-		List<String> files = new ArrayList<String>();
-		files = getAllFiles(copyToPath, 0); 
-		return files;
-	}
-	
-	@Override
-	public void nextTuple() {
-		LOG.info("Releasing nextTuple");
-		List<String> files = listFiles();
-		LOG.info("Files returned: " + files.size());
-		String typeOfFile = configContext.getString("dataSourceConfig.fileFormat");
-		LOG.info("typeOfFile returned: " + typeOfFile);
-		
-		for(String fileName:files){
-			LOG.info("fileName: " + fileName);
-			LOG.info("processFileMap.get(fileName): " + processFileMap.get(fileName));
-			if(processFileMap.get(fileName) == null || processFileMap.get(fileName) == false){
-				processFileMap.put(fileName, true);
-				BufferedReader br = null; 
-				Reader decoder = null;
-				GZIPInputStream in = null; 
-				InputStream inStream = null;
-				
-				try{
-					if(typeOfFile.equalsIgnoreCase("GZIP")){
-						in = new GZIPInputStream(new FileInputStream(new File(fileName)));
-						decoder = new InputStreamReader(in);
-					}else if(typeOfFile.equalsIgnoreCase("CSV")){
-						inStream = new FileInputStream(new File(fileName)); 
-						decoder = new InputStreamReader(inStream);
-					}else{
-						LOG.error("No known file type specified");
-						continue;
-					}
-					
-					br = new BufferedReader(decoder);
-					int lineNo = 0; 
-					String line = "";
-					while((line = br.readLine())!= null){
-						++lineNo;
-			        	//String line = br.readLine();
-			        	//loggerHDFSSpout.info("line number " + lineNo + "is: " + line);
-			        	//if(line == null || line.equalsIgnoreCase(""))
-			        	//	break;
-			        	LOG.info("Emitting line from file: " + fileName);
-			        	//_collector.emit(new ValuesArray(line), lineNo);
-                        _collector.emit(Arrays.asList((Object)line));
-			        	LOG.info("Emitted line no: " + lineNo + " and line: " + line);
-						Utils.sleep(100);
-					}
-				}
-				catch (Exception e) {
-					// TODO: handle exception
-					e.printStackTrace();
-				}finally{
-					try {
-						if(br != null)
-							br.close();
-						if(decoder != null)
-							decoder.close();
-						if(in != null)
-							in.close();
-						if(inStream != null)
-							inStream.close();
-					} catch (IOException e) {
-						// TODO Auto-generated catch block
-						e.printStackTrace();
-					}
-				}
-			}else{
-				LOG.info("Processed the files before, already done! ");
-				//Utils.sleep(10000);
-			}
-			
-		}
-		
-	}
-	
-	public void fail(Object msgId) {
-	    int transactionId = (Integer) msgId;
-	    LOG.info(transactionId + " failed");
-	}
-	
-	public void ack(Object msgId) {
-	    int transactionId = (Integer) msgId;
-	    LOG.info(transactionId + " acknowledged");
-	}
-
-	@Override
-	public void open(Map arg0, TopologyContext context,
-			SpoutOutputCollector collector) {
-		 _collector = collector;
-		 _context = context;
-	}
-
-	@Override
-	public void declareOutputFields(OutputFieldsDeclarer declarer) {
-		// TODO Auto-generated method stub
-		declarer.declare(new Fields("line"));
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java
deleted file mode 100644
index e9ba3ca..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/hdfs/HDFSSourcedStormSpoutProvider.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.dataproc.impl.storm.hdfs;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.dataproc.impl.storm.StormSpoutProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import backtype.storm.topology.base.BaseRichSpout;
-
-public class HDFSSourcedStormSpoutProvider implements StormSpoutProvider {
-	private static final Logger LOG = LoggerFactory.getLogger(HDFSSourcedStormSpoutProvider.class);
-	
-	public abstract static class HDFSSpout extends BaseRichSpout{
-		public abstract void copyFiles(); 
-		public void fail(Object msgId) {
-		    int transactionId = (Integer) msgId;
-		    LOG.info(transactionId + " failed");
-		}
-		
-		public void ack(Object msgId) {
-		    int transactionId = (Integer) msgId;
-		    LOG.info(transactionId + " acknowledged");
-		}
-		
-		public static HDFSSpout getHDFSSpout(String conf, Config configContext){
-			if(conf.equalsIgnoreCase("data collection")){
-				return new DataCollectionHDFSSpout(configContext); 
-			}
-			return null;
-		}
-	}
-	
-	@Override
-	public BaseRichSpout getSpout(Config context){
-		LOG.info("GetHDFSSpout called");
-		String typeOperation = context.getString("dataSourceConfig.typeOperation");
-		HDFSSpout spout = HDFSSpout.getHDFSSpout(typeOperation, context);
-		spout.copyFiles();
-		return spout;
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
deleted file mode 100644
index b9d998d..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/KafkaSpoutProvider.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- *
- *  *
- *  *    Licensed to the Apache Software Foundation (ASF) under one or more
- *  *    contributor license agreements.  See the NOTICE file distributed with
- *  *    this work for additional information regarding copyright ownership.
- *  *    The ASF licenses this file to You under the Apache License, Version 2.0
- *  *    (the "License"); you may not use this file except in compliance with
- *  *    the License.  You may obtain a copy of the License at
- *  *
- *  *       http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  *    Unless required by applicable law or agreed to in writing, software
- *  *    distributed under the License is distributed on an "AS IS" BASIS,
- *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  *    See the License for the specific language governing permissions and
- *  *    limitations under the License.
- *  *
- *
- */
-
-package org.apache.eagle.dataproc.impl.storm.kafka;
-
-import backtype.storm.spout.Scheme;
-import backtype.storm.spout.SchemeAsMultiScheme;
-import backtype.storm.topology.base.BaseRichSpout;
-import com.typesafe.config.Config;
-import org.apache.eagle.dataproc.impl.storm.StormSpoutProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import storm.kafka.BrokerHosts;
-import storm.kafka.KafkaSpout;
-import storm.kafka.SpoutConfig;
-import storm.kafka.ZkHosts;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Since 6/8/16.
- */
-public class KafkaSpoutProvider implements StormSpoutProvider {
-    private final static Logger LOG = LoggerFactory.getLogger(KafkaSpoutProvider.class);
-    private final static String DEFAULT_CONFIG_PREFIX = "dataSourceConfig";
-    private final static String DEFAULT_CONSUMER_GROUP_ID = "eagleConsumer";
-    private final static String DEFAULT_TRANSACTION_ZK_ROOT = "/consumers";
-
-    private String configPrefix = DEFAULT_CONFIG_PREFIX;
-
-    public KafkaSpoutProvider(){}
-
-    public KafkaSpoutProvider(String prefix){
-        this.configPrefix = prefix;
-    }
-
-    @Override
-    public BaseRichSpout getSpout(Config config){
-        Config context = config;
-        if(this.configPrefix!=null) context = config.getConfig(configPrefix);
-
-        // the following is for fetching data from one topic
-        // Kafka topic
-        String topic = context.getString("topic");
-        // Kafka broker zk connection
-        String zkConnString = context.getString("zkConnection");
-        // Kafka fetch size
-        int fetchSize = context.hasPath("fetchSize") ? context.getInt("fetchSize") : 1048586;
-        LOG.info(String.format("Use topic : %s, zkConnection : %s , fetchSize : %d", topic, zkConnString, fetchSize));
-
-        /*
-         the following is for recording offset for processing the data
-         the zk path to store current offset is comprised of the following
-         offset zkPath = zkRoot + "/" + topic + "/" + consumerGroupId + "/" + partition_Id
-
-         consumerGroupId is for differentiating different consumers which consume the same topic
-        */
-        // transaction zkRoot
-        String zkRoot = context.hasPath("transactionZKRoot") ? context.getString("transactionZKRoot") : DEFAULT_TRANSACTION_ZK_ROOT;
-        // Kafka consumer group id
-        String groupId = context.hasPath("consumerGroupId") ? context.getString("consumerGroupId") : DEFAULT_CONSUMER_GROUP_ID;
-        String brokerZkPath = context.hasPath("brokerZkPath") ? context.getString("brokerZkPath") : null;
-        BrokerHosts hosts;
-        if(brokerZkPath == null) {
-            hosts = new ZkHosts(zkConnString);
-        } else {
-            hosts = new ZkHosts(zkConnString, brokerZkPath);
-        }
-
-        SpoutConfig spoutConfig = new SpoutConfig(hosts,
-                topic,
-                zkRoot + "/" + topic,
-                groupId);
-
-        // transaction zkServers
-        String[] txZkServers = context.hasPath("txZkServers") ? context.getString("txZkServers").split(",") : new String[]{"localhost:2181"};
-        spoutConfig.zkServers = Arrays.asList(txZkServers).stream().map(server -> server.split(":")[0]).collect(Collectors.toList());
-        // transaction zkPort
-        spoutConfig.zkPort = Integer.parseInt(txZkServers[0].split(":")[1]);
-        LOG.info("txZkServers:" + spoutConfig.zkServers + ", zkPort:" + spoutConfig.zkPort);
-        // transaction update interval
-        spoutConfig.stateUpdateIntervalMs = context.hasPath("transactionStateUpdateMS") ? context.getLong("transactionStateUpdateMS") : 2000;
-        // Kafka fetch size
-        spoutConfig.fetchSizeBytes = fetchSize;
-        // "startOffsetTime" is for test usage, prod should not use this
-        if (context.hasPath("startOffsetTime")) {
-            spoutConfig.startOffsetTime = context.getInt("startOffsetTime");
-        }
-        // "forceFromStart" is for test usage, prod should not use this
-        if (context.hasPath("forceFromStart")) {
-            spoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
-        }
-
-        if (context.hasPath("schemeCls")) {
-            try {
-                Scheme s = (Scheme)Class.forName(context.getString("schemeCls")).newInstance();
-                spoutConfig.scheme = new SchemeAsMultiScheme(s);
-            }catch(Exception ex){
-                LOG.error("error instantiating scheme object");
-                throw new IllegalStateException(ex);
-            }
-        }else{
-            String err = "schemeCls must be present";
-            LOG.error(err);
-            throw new IllegalStateException(err);
-        }
-        return new KafkaSpout(spoutConfig);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/SpoutKafkaMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/SpoutKafkaMessageDeserializer.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/SpoutKafkaMessageDeserializer.java
deleted file mode 100644
index 76ca458..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/kafka/SpoutKafkaMessageDeserializer.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.impl.storm.kafka;
-
-import java.io.Serializable;
-
-public interface SpoutKafkaMessageDeserializer extends Serializable{
-	public Object deserialize(byte[] arg0);
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java
deleted file mode 100644
index 9c1ab68..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/partition/CustomPartitionGrouping.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- *
- *    Licensed to the Apache Software Foundation (ASF) under one or more
- *    contributor license agreements.  See the NOTICE file distributed with
- *    this work for additional information regarding copyright ownership.
- *    The ASF licenses this file to You under the Apache License, Version 2.0
- *    (the "License"); you may not use this file except in compliance with
- *    the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- *
- */
-
-package org.apache.eagle.dataproc.impl.storm.partition;
-
-import backtype.storm.generated.GlobalStreamId;
-import backtype.storm.grouping.CustomStreamGrouping;
-import backtype.storm.task.WorkerTopologyContext;
-import org.apache.eagle.partition.PartitionStrategy;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class CustomPartitionGrouping implements CustomStreamGrouping {
-
-    public List<Integer> targetTasks;
-    public PartitionStrategy strategy;
-
-    public CustomPartitionGrouping(PartitionStrategy strategy) {
-        this.strategy = strategy;
-    }
-
-    @Override
-    public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) {
-        this.targetTasks = new ArrayList<>(targetTasks);
-    }
-
-    @Override
-    public List<Integer> chooseTasks(int taskId, List<Object> values) {
-        int numTasks = targetTasks.size();
-        int targetTaskIndex = strategy.balance((String)values.get(0), numTasks);
-        return Arrays.asList(targetTasks.get(targetTaskIndex));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/zookeeper/ZKStateConfig.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/zookeeper/ZKStateConfig.java b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/zookeeper/ZKStateConfig.java
deleted file mode 100644
index f9515f5..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/java/org/apache/eagle/dataproc/impl/storm/zookeeper/ZKStateConfig.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.impl.storm.zookeeper;
-
-import java.io.Serializable;
-
-public class ZKStateConfig implements Serializable {
-    private static final long serialVersionUID = 1L;
-    public String zkQuorum;
-    public String zkRoot;
-    public int zkSessionTimeoutMs;
-    public int zkRetryTimes;
-    public int zkRetryInterval;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/application.conf b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/application.conf
deleted file mode 100644
index c386a71..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/application.conf
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
-  "envContextConfig" : {
-    "env" : "storm",
-    "mode" : "local",
-    "topologyName" : "SpadesMonitorTopology",
-    "stormConfigFile" : "spades-monitor-storm.yaml",
-    "parallelismConfig" : {
-      "SpadesMonitorStream" : 1,
-      "SpadesMonitorExecutor*" : 1
-    }
-  },
-  "dataSourceConfig": {
-    "topic" : "spades_monitor_sandbox",
-    "zkConnection" : "sandbox.hortonworks.com:2181",
-    "zkConnectionTimeoutMS" : 15000,
-    "consumerGroupId" : "eagle.consumer",
-    "fetchSize" : 1048586,
-    "transactionZKServers" : "sandbox.hortonworks.com",
-    "transactionZKPort" : 2181,
-    "transactionZKRoot" : "/consumers",
-    "transactionStateUpdateMS" : 2000
-  },
-  "alertExecutorConfigs" : {
-    "SpadesMonitorExecutor" : {
-      "parallelism" : 1,
-      "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner"
-      "needValidation" : "true"
-    }
-  },
-  "eagleProps" : {
-    "site" : "sandbox",
-    "application": "SpadesMonitor",
-    "dataJoinPollIntervalSec" : 30,
-    "mailHost" : "mailHost.com",
-    "mailSmtpPort":"25",
-    "mailDebug" : "true",
-    "balancePartitionEnabled" : true,
-    #"partitionRefreshIntervalInMin" : 60,
-    #"kafkaStatisticRangeInMin" : 60,
-    "eagleService": {
-      "host": "localhost",
-      "port": 9099,
-      "username": "admin",
-      "password": "secret"
-    }
-    "readHdfsUserCommandPatternFrom" : "file"
-  },
-  "dynamicConfigSource" : {
-    "enabled" : true,
-    "initDelayMillis" : 0,
-    "delayMillis" : 30000
-  },
-  "eagleNotificationProps" : {
-    "eagleStoreEnabled": true,
-    "kafka_broker":"127.0.0.1:6667"
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/log4j.properties b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/log4j.properties
deleted file mode 100644
index d59ded6..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, stdout
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala
deleted file mode 100644
index d4836e4..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/utils/NameConstants.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream.utils
-
-object NameConstants {
-  val FIELD_PREFIX = "f"
-  val FIELD_KEY = "key"
-  val FIELD_VALUE = "value"
-  val FIELD_SEPARATOR = "_"
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/pom.xml b/eagle-core/eagle-data-process/eagle-stream-process-base/pom.xml
deleted file mode 100644
index dd186d6..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/pom.xml
+++ /dev/null
@@ -1,128 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <groupId>org.apache.eagle</groupId>
-        <artifactId>eagle-data-process-parent</artifactId>
-        <version>0.5.0-incubating-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-
-    <modelVersion>4.0.0</modelVersion>
-    <artifactId>eagle-stream-process-base</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-math3</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_${scala.version}</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.typesafe</groupId>
-            <artifactId>config</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-common</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_${scala.version}</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <!--<dependency>-->
-            <!--<groupId>org.apache.storm</groupId>-->
-            <!--<artifactId>storm-core</artifactId>-->
-            <!--<exclusions>-->
-                <!--<exclusion>-->
-                    <!--<groupId>ch.qos.logback</groupId>-->
-                    <!--<artifactId>logback-classic</artifactId>-->
-                <!--</exclusion>-->
-                <!--<exclusion>-->
-                    <!--<groupId>log4j</groupId>-->
-                    <!--<artifactId>log4j</artifactId>-->
-                <!--</exclusion>-->
-                <!--<exclusion>-->
-                    <!--<groupId>org.slf4j</groupId>-->
-                    <!--<artifactId>log4j-over-slf4j</artifactId>-->
-                <!--</exclusion>-->
-            <!--</exclusions>-->
-        <!--</dependency>-->
-    </dependencies>
-
-    <build>
-    <plugins>
-        <plugin>
-            <groupId>org.scala-tools</groupId>
-            <artifactId>maven-scala-plugin</artifactId>
-            <executions>
-                <execution>
-                    <id>scala-compile-first</id>
-                    <phase>process-resources</phase>
-                    <goals>
-                        <goal>add-source</goal>
-                        <goal>compile</goal>
-                    </goals>
-                </execution>
-                <execution>
-                    <id>scala-test-compile</id>
-                    <phase>process-test-resources</phase>
-                    <goals>
-                        <goal>testCompile</goal>
-                    </goals>
-                </execution>
-            </executions>
-        </plugin>
-
-        <plugin>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest-maven-plugin</artifactId>
-            <configuration>
-                <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-                <junitxml>.</junitxml>
-                <filereports>TestSuite.txt</filereports>
-            </configuration>
-            <executions>
-                <execution>
-                    <id>test</id>
-                    <goals>
-                        <goal>test</goal>
-                    </goals>
-                </execution>
-            </executions>
-        </plugin>
-    </plugins>
-    </build>
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/EagleOutputCollector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/EagleOutputCollector.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/EagleOutputCollector.java
deleted file mode 100644
index 0c4b1ab..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/EagleOutputCollector.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.core;
-
-import java.io.Serializable;
-
-/**
- * expose simple interface for streaming executor to populate output data
- *
- */
-public interface EagleOutputCollector extends Serializable{
-	void collect(ValuesArray t);
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/JsonSerDeserUtils.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/JsonSerDeserUtils.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/JsonSerDeserUtils.java
deleted file mode 100644
index 5df7e55..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/JsonSerDeserUtils.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.core;
-
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-public class JsonSerDeserUtils {
-	private static final Logger LOG = LoggerFactory.getLogger(JsonSerDeserUtils.class);
-
-	public static <T> String serialize(T o) throws Exception{
-		return serialize(o, null);
-	}
-	
-	public static <T> String serialize(T o, List<Module> modules) throws Exception {
-		ObjectMapper mapper = new ObjectMapper();
-		if (modules != null) { 
-			mapper.registerModules(modules);
-		}
-		return mapper.writeValueAsString(o);
-	}
-
-	public static <T> T deserialize(String value, Class<T> cls) throws Exception{
-		return deserialize(value, cls, null);
-	}
-	
-	public static <T> T deserialize(String value, Class<T> cls, List<Module> modules) throws Exception{
-		ObjectMapper mapper = new ObjectMapper();
-		if (modules != null) { 
-			mapper.registerModules(modules);
-		}
-		return mapper.readValue(value, cls);	
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/ValuesArray.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/ValuesArray.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/ValuesArray.java
deleted file mode 100644
index 9971cb2..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/core/ValuesArray.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.core;
-
-import java.util.ArrayList;
-
-/**
- * multiple datapoints are stored within one ValuesArray object
- * sent out
- */
-public class ValuesArray extends ArrayList<Object>{
-	private static final long serialVersionUID = -8218427810421668178L;
-
-	public ValuesArray() {
-        
-    }
-    
-    public ValuesArray(Object... vals) {
-        super(vals.length);
-        for(Object o: vals) {
-            add(o);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/AbstractConfigOptionParser.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/AbstractConfigOptionParser.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/AbstractConfigOptionParser.java
deleted file mode 100644
index f40bb76..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/AbstractConfigOptionParser.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.util;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import org.apache.commons.cli.*;
-
-import java.util.Map;
-
-/**
- * @since 8/22/15
- */
-public abstract class AbstractConfigOptionParser {
-
-    // private final Options options;
-    private final Parser parser;
-
-    public AbstractConfigOptionParser(){
-        parser = parser();
-        //options = options();
-    }
-
-    /**
-     * @return Parser
-     */
-    protected abstract Parser parser();
-
-    /**
-     * @return Options
-     */
-    protected abstract Options options();
-
-    public abstract Map<String,String> parseConfig(String[] arguments) throws ParseException;
-
-    /**
-     * Load config as system properties
-     *
-     * @param arguments command line arguments
-     * @throws ParseException
-     */
-    public Config load(String[] arguments) throws ParseException {
-        Map<String,String> configProps = parseConfig(arguments);
-        for(Map.Entry<String,String> entry:configProps.entrySet()){
-            System.setProperty(entry.getKey(),entry.getValue());
-        }
-        System.setProperty("config.trace", "loads");
-        return ConfigFactory.load();
-    }
-
-    public CommandLine parse(String[] arguments) throws ParseException {
-        return this.parser.parse(this.options(),arguments);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/ConfigOptionParser.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/ConfigOptionParser.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/ConfigOptionParser.java
deleted file mode 100644
index bbd4e38..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/dataproc/util/ConfigOptionParser.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.dataproc.util;
-
-import org.apache.commons.cli.*;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-public class ConfigOptionParser extends AbstractConfigOptionParser {
-    private final static String CONFIG_OPT_FLAG = "D";
-
-    @Override
-    protected Parser parser() {
-        return new BasicParser();
-    }
-
-    @Override
-    protected Options options() {
-        Options options = new Options();
-        options.addOption(CONFIG_OPT_FLAG, true, "Config properties in format of \"-D key=value\"");
-        return options;
-    }
-
-    @Override
-    public Map<String,String> parseConfig(String[] arguments) throws ParseException {
-        CommandLine cmd = parse(arguments);
-        return parseCommand(cmd);
-    }
-
-    protected Map<String,String> parseCommand(CommandLine cmd) throws ParseException {
-        Map<String,String> result = new HashMap<>();
-        if(cmd.hasOption(CONFIG_OPT_FLAG)){
-            String[] values = cmd.getOptionValues(CONFIG_OPT_FLAG);
-            for(String value:values){
-                int eqIndex = value.indexOf("=");
-                if(eqIndex>0 && eqIndex<value.length()){
-                    String k = value.substring(0,eqIndex);
-                    String v = value.substring(eqIndex+1,value.length());
-                    if(result.containsKey(k)){
-                        throw new ParseException("Duplicated "+CONFIG_OPT_FLAG+" "+value);
-                    }else{
-                        result.put(k,v);
-                    }
-                }else{
-                    throw new ParseException("Invalid format: -"+CONFIG_OPT_FLAG+" "+value+", required: -"+CONFIG_OPT_FLAG+" key=value");
-                }
-            }
-        }
-        return result;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/DataDistributionDao.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/DataDistributionDao.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/DataDistributionDao.java
deleted file mode 100644
index 0b17775..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/DataDistributionDao.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- *
- *    Licensed to the Apache Software Foundation (ASF) under one or more
- *    contributor license agreements.  See the NOTICE file distributed with
- *    this work for additional information regarding copyright ownership.
- *    The ASF licenses this file to You under the Apache License, Version 2.0
- *    (the "License"); you may not use this file except in compliance with
- *    the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- *
- */
-
-package org.apache.eagle.partition;
-
-import java.io.Serializable;
-import java.util.List;
-
-public interface DataDistributionDao extends Serializable {
-
-    List<Weight> fetchDataDistribution(long startTime, long endTime) throws Exception;
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionAlgorithm.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionAlgorithm.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionAlgorithm.java
deleted file mode 100644
index 0614388..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionAlgorithm.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- *
- *    Licensed to the Apache Software Foundation (ASF) under one or more
- *    contributor license agreements.  See the NOTICE file distributed with
- *    this work for additional information regarding copyright ownership.
- *    The ASF licenses this file to You under the Apache License, Version 2.0
- *    (the "License"); you may not use this file except in compliance with
- *    the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- *
- */
-
-package org.apache.eagle.partition;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public interface PartitionAlgorithm extends Serializable {
-    Map<String, Integer> partition(List<Weight> weights, int k);
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategy.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategy.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategy.java
deleted file mode 100644
index e431f28..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategy.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- *
- *    Licensed to the Apache Software Foundation (ASF) under one or more
- *    contributor license agreements.  See the NOTICE file distributed with
- *    this work for additional information regarding copyright ownership.
- *    The ASF licenses this file to You under the Apache License, Version 2.0
- *    (the "License"); you may not use this file except in compliance with
- *    the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- *
- */
-
-package org.apache.eagle.partition;
-
-import java.io.Serializable;
-
-public interface PartitionStrategy extends Serializable {
-
-    int balance(String key, int buckNum);
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategyImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategyImpl.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategyImpl.java
deleted file mode 100644
index eacefd5..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/PartitionStrategyImpl.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- *
- *    Licensed to the Apache Software Foundation (ASF) under one or more
- *    contributor license agreements.  See the NOTICE file distributed with
- *    this work for additional information regarding copyright ownership.
- *    The ASF licenses this file to You under the Apache License, Version 2.0
- *    (the "License"); you may not use this file except in compliance with
- *    the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- *
- */
-
-package org.apache.eagle.partition;
-
-import org.apache.commons.lang3.time.DateUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-public class PartitionStrategyImpl implements PartitionStrategy {
-
-    public DataDistributionDao dao;
-    public PartitionAlgorithm algorithm;
-    public Map<String, Integer> routingTable;
-    public long lastRefreshTime;
-    public long refreshInterval;
-    public long timeRange;
-    public static long DEFAULT_TIME_RANGE = 2 * DateUtils.MILLIS_PER_DAY;
-    public static long DEFAULT_REFRESH_INTERVAL = 2 * DateUtils.MILLIS_PER_HOUR;
-    private final Logger LOG = LoggerFactory.getLogger(PartitionStrategyImpl.class);
-
-    public PartitionStrategyImpl(DataDistributionDao dao, PartitionAlgorithm algorithm, long refreshInterval, long timeRange) {
-        this.dao = dao;
-        this.algorithm = algorithm;
-        this.refreshInterval = refreshInterval;
-        this.timeRange = timeRange;
-    }
-
-    public PartitionStrategyImpl(DataDistributionDao dao, PartitionAlgorithm algorithm) {
-        this(dao, algorithm, DEFAULT_REFRESH_INTERVAL, DEFAULT_TIME_RANGE);
-    }
-
-    public boolean needRefresh() {
-        if (System.currentTimeMillis() > lastRefreshTime + refreshInterval) {
-            lastRefreshTime = System.currentTimeMillis();
-            return true;
-        }
-        return false;
-    }
-
-    public Map<String, Integer> generateRoutingTable(int buckNum) {
-        try {
-            long currentTime = System.currentTimeMillis();
-            List<Weight> weights = dao.fetchDataDistribution(currentTime - timeRange, currentTime);
-            routingTable = algorithm.partition(weights, buckNum);
-            return routingTable;
-        }
-        catch (Exception ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    @Override
-    public int balance(String key, int buckNum) {
-        if (needRefresh()) {
-            LOG.info("Going to refresh routing table");
-            routingTable = generateRoutingTable(buckNum);
-            LOG.info("Finish refresh routing table");
-        }
-        if (routingTable.containsKey(key)) {
-            return routingTable.get(key);
-        }
-        else {
-            return Math.abs(key.hashCode()) % buckNum;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/Weight.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/Weight.java b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/Weight.java
deleted file mode 100644
index 14d005d..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/java/org/apache/eagle/partition/Weight.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- *    Licensed to the Apache Software Foundation (ASF) under one or more
- *    contributor license agreements.  See the NOTICE file distributed with
- *    this work for additional information regarding copyright ownership.
- *    The ASF licenses this file to You under the Apache License, Version 2.0
- *    (the "License"); you may not use this file except in compliance with
- *    the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- *
- */
-
-package org.apache.eagle.partition;
-
-public class Weight {
-    public String key;
-    public Double value;
-
-    public Weight(String key, Double value) {
-        this.key = key;
-        this.value = value;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/Collector.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/Collector.scala b/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/Collector.scala
deleted file mode 100644
index 5361001..0000000
--- a/eagle-core/eagle-data-process/eagle-stream-process-base/src/main/scala/org/apache/eagle/datastream/Collector.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.datastream
-
-trait Collector[-R] {
-  def collect(r : R);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b31bac50/eagle-core/eagle-data-process/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/pom.xml b/eagle-core/eagle-data-process/pom.xml
index 897cbf8..a90b1f0 100644
--- a/eagle-core/eagle-data-process/pom.xml
+++ b/eagle-core/eagle-data-process/pom.xml
@@ -23,11 +23,135 @@
         <artifactId>eagle-core</artifactId>
         <version>0.5.0-incubating-SNAPSHOT</version>
     </parent>
-    <artifactId>eagle-data-process-parent</artifactId>
-    <packaging>pom</packaging>
+    <artifactId>eagle-data-process</artifactId>
+    <packaging>jar</packaging>
     <description>Eagle Data Process Framework</description>
-    <modules>
-        <module>eagle-stream-process-base</module>
-        <module>eagle-stream-process-api</module>
-    </modules>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.storm</groupId>
+            <artifactId>storm-core</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-classic</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>log4j-over-slf4j</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.storm</groupId>
+            <artifactId>storm-kafka</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-classic</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>log4j-over-slf4j</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-reflect</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-libaray</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_${scala.version}</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+            <version>${kafka-clients.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.typesafe</groupId>
+            <artifactId>config</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka_2.11</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>scala-compile-first</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <skipTests>true</skipTests>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.scalatest</groupId>
+                <artifactId>scalatest-maven-plugin</artifactId>
+                <configuration>
+                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+                    <junitxml>.</junitxml>
+                    <filereports>TestSuite.txt</filereports>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>test</id>
+                        <goals>
+                            <goal>test</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 </project>



Mime
View raw message