eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject incubator-eagle git commit: wrapper alert engine as one application of eagle server hdfs audit log + alert engine end-2-end
Date Fri, 26 Aug 2016 04:35:45 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/develop 36f9a8481 -> 000b74604


wrapper alert engine as one application of eagle server
hdfs audit log + alert engine end-2-end

https://issues.apache.org/jira/browse/EAGLE-481

Author: @yonzhang <yonzhang2012@apache.org>

Closes: #392


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/000b7460
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/000b7460
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/000b7460

Branch: refs/heads/develop
Commit: 000b746041ec07774e4ed5d5562f9c66307f1c20
Parents: 36f9a84
Author: yonzhang <yonzhang2012@gmail.com>
Authored: Thu Aug 25 21:39:53 2016 -0700
Committer: yonzhang <yonzhang2012@gmail.com>
Committed: Thu Aug 25 21:39:53 2016 -0700

----------------------------------------------------------------------
 .../eagle-alert-parent/eagle-alert-app/pom.xml  |  35 ++++
 .../eagle/alert/app/AlertUnitTopologyApp.java   |  39 ++++
 .../alert/app/AlertUnitTopologyAppProvider.java |  29 +++
 ...e.alert.app.AlertUnitTopologyAppProvider.xml | 135 +++++++++++++
 ...org.apache.eagle.app.spi.ApplicationProvider |  18 ++
 .../src/main/resources/application.conf         |  60 ++++++
 .../alert/AlertPolicyValidateProvider.java      |  36 ----
 .../eagle/alert/config/ZKConfigBuilder.java     |   8 +-
 .../scheme/JsonStringStreamNameSelector.java    |   6 +-
 .../alert/engine/spout/CorrelationSpout.java    |   4 +-
 .../src/main/resources/application.conf         |  19 +-
 .../alert/engine/e2e/CoordinatorClient.java     |   6 +-
 eagle-core/eagle-alert-parent/pom.xml           |   9 +-
 .../eagle/app/service/ApplicationContext.java   |   6 +-
 .../src/main/resources/log4j.properties         |   2 +-
 .../eagle-security-hdfs-auditlog/README.md      | 191 +++++++++++++++++++
 ...ecurity.auditlog.HdfsAuditLogAppProvider.xml |  26 ++-
 .../src/main/resources/scripts.txt              |  22 +++
 eagle-server/pom.xml                            |   7 +
 19 files changed, 581 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
new file mode 100644
index 0000000..768709f
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.eagle</groupId>
+    <artifactId>eagle-alert-parent</artifactId>
+    <version>0.5.0-incubating-SNAPSHOT</version>
+  </parent>
+  <artifactId>eagle-alert-app</artifactId>
+  <packaging>jar</packaging>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.eagle</groupId>
+      <artifactId>eagle-app-base</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java
new file mode 100644
index 0000000..a80c3b8
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyApp.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.alert.app;import backtype.storm.generated.StormTopology;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.alert.engine.UnitTopologyMain;
+import org.apache.eagle.app.StormApplication;
+import org.apache.eagle.app.environment.impl.StormEnvironment;
+
+/**
+ * since 8/25/16.
+ */
+public class AlertUnitTopologyApp extends StormApplication {
+    @Override
+    public StormTopology execute(Config config, StormEnvironment environment) {
+        return UnitTopologyMain.createTopology(config);
+    }
+
+    public static void main(String[] args){
+        Config config = ConfigFactory.load();
+        AlertUnitTopologyApp app = new AlertUnitTopologyApp();
+        app.run(config);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java
new file mode 100644
index 0000000..39a4583
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/java/org/apache/eagle/alert/app/AlertUnitTopologyAppProvider.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.alert.app;
+import org.apache.eagle.app.spi.AbstractApplicationProvider;
+
+/**
+ * since 8/25/16.
+ */
+public class AlertUnitTopologyAppProvider extends AbstractApplicationProvider<AlertUnitTopologyApp>
{
+    @Override
+    public AlertUnitTopologyApp getApplication() {
+        return new AlertUnitTopologyApp();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
new file mode 100644
index 0000000..498cb8d
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<application>
+    <type>AlertUnitTopologyApp</type>
+    <name>Alert Unit Topology Application</name>
+    <version>0.5.0-incubating</version>
+    <appClass>org.apache.eagle.alert.app.AlertUnitTopologyApp</appClass>
+    <viewPath>/apps/alert</viewPath>
+    <configuration>
+       <!-- alert topology sizing parameters -->
+        <property>
+            <name>topology.numOfTotalWorkers</name>
+            <displayName>topology.numOfTotalWorkers</displayName>
+            <value>2</value>
+            <description>number of total storm workers</description>
+        </property>
+        <property>
+            <name>topology.numOfSpoutTasks</name>
+            <displayName>topology.numOfSpoutTasks</displayName>
+            <value>1</value>
+            <description>number of spout tasks</description>
+        </property>
+        <property>
+            <name>topology.numOfRouterBolts</name>
+            <displayName>topology.numOfRouterBolts</displayName>
+            <value>4</value>
+            <description>number of router tasks</description>
+        </property>
+        <property>
+            <name>topology.numOfAlertBolts</name>
+            <displayName>topology.numOfAlertBolts</displayName>
+            <value>10</value>
+            <description>number of alert tasks</description>
+        </property>
+        <property>
+            <name>topology.numOfPublishTasks</name>
+            <displayName>topology.numOfPublishTasks</displayName>
+            <value>1</value>
+            <description>number of publish tasks</description>
+        </property>
+        <property>
+            <name>topology.messageTimeoutSecs</name>
+            <displayName>topology.messageTimeoutSecs</displayName>
+            <value>3600</value>
+            <description>number of tuple timeout in seconds</description>
+        </property>
+
+        <!-- alert spout configuration -->
+        <property>
+            <name>spout.kafkaBrokerZkQuorum</name>
+            <displayName>spout.kafkaBrokerZkQuorum</displayName>
+            <value>server.eagle.apache.org:2181</value>
+            <description>zookeeper quorum for spout to consume data</description>
+        </property>
+        <property>
+            <name>spout.kafkaBrokerZkBasePath</name>
+            <displayName>spout.kafkaBrokerZkBasePath</displayName>
+            <value>/brokers</value>
+            <description>zk znode path for kafka brokers</description>
+        </property>
+        <property>
+            <name>spout.stormKafkaUseSameZkQuorumWithKafkaBroker</name>
+            <displayName>spout.stormKafkaUseSameZkQuorumWithKafkaBroker</displayName>
+            <value>true</value>
+            <description>same zookeeper for kafka server and kafka consumer(Storm-Kafka)</description>
+        </property>
+        <property>
+            <name>spout.stormKafkaTransactionZkPath</name>
+            <displayName>spout.stormKafkaTransactionZkPath</displayName>
+            <value>/consumers</value>
+            <description>zk path for storm kafka transaction</description>
+        </property>
+        <property>
+            <name>spout.stormKafkaEagleConsumer</name>
+            <displayName>spout.stormKafkaEagleConsumer</displayName>
+            <value>eagle_consumer</value>
+            <description>zookeeper quorum for spout to consume data</description>
+        </property>
+
+        <!-- zk config for alert engine -->
+        <property>
+            <name>zkConfig.zkQuorum</name>
+            <displayName>zkConfig.zkQuorum</displayName>
+            <value>server.eagle.apache.org:2181</value>
+            <description>zk quorum for alert engine</description>
+        </property>
+        <property>
+            <name>zkConfig.zkRoot</name>
+            <displayName>zkConfig.zkRoot</displayName>
+            <value>/alert</value>
+            <description>zk znode path for alert engine</description>
+        </property>
+
+        <property>
+            <name>metadataService.context</name>
+            <displayName>metadataService.context</displayName>
+            <value>/rest</value>
+            <description>metadata service context path</description>
+        </property>
+        <property>
+            <name>metadataService.host</name>
+            <displayName>metadataService.host</displayName>
+            <value>localhost</value>
+            <description>metadata service host</description>
+        </property>
+        <property>
+            <name>metadataService.port</name>
+            <displayName>metadataService.port</displayName>
+            <value>9090</value>
+            <description>metadata service port</description>
+        </property>
+    </configuration>
+    <docs>
+        <install>
+        </install>
+        <uninstall>
+        </uninstall>
+    </docs>
+</application>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
new file mode 100644
index 0000000..ceb079d
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/services/org.apache.eagle.app.spi.ApplicationProvider
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.eagle.alert.app.AlertUnitTopologyAppProvider

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
new file mode 100644
index 0000000..1a25cfa
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/application.conf
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{
+  "appId" : "HBaseAuditLogApp",
+  "mode" : "LOCAL",
+  "siteId" : "testsite",
+  "topology" : {
+    "name" : "alertUnitTopology_1",
+    "numOfTotalWorkers" : 2,
+    "numOfSpoutTasks" : 1,
+    "numOfRouterBolts" : 4,
+    "numOfAlertBolts" : 10,
+    "numOfPublishTasks" : 1,
+    "messageTimeoutSecs": 3600,
+    "localMode" : "true"
+  },
+  "spout" : {
+    "kafkaBrokerZkQuorum": "server.eagle.apache.org:2181",
+    "kafkaBrokerZkBasePath": "/kafka",
+    "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
+    "stormKafkaTransactionZkQuorum": "",
+    "stormKafkaTransactionZkPath": "/consumers",
+    "stormKafkaEagleConsumer": "eagle_consumer"
+  },
+  "zkConfig" : {
+    "zkQuorum" : "server.eagle.apache.org:2181",
+    "zkRoot" : "/alert"
+  },
+  "metadataService": {
+    "context" : "/rest",
+    "host" : "localhost",
+    "port" : 9090
+  },
+  "metric":{
+    "sink": {
+      // "kafka": {
+      //  "topic": "alert_metric"
+      //  "bootstrap.servers": "localhost:6667"
+      // }
+      //      "stdout": {}
+      //      "elasticsearch": {
+      //        "hosts": ["localhost:9200"]
+      //        "index": "alert_metric"
+      //        "timestampField": "timestamp"
+      //      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java
b/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java
deleted file mode 100644
index 47ddcb9..0000000
--- a/eagle-core/eagle-alert-parent/eagle-alert-service/src/main/java/org/apache/eagle/service/alert/AlertPolicyValidateProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.service.alert;
-
-import java.util.List;
-
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import com.fasterxml.jackson.databind.Module;
-
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible=true)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public abstract class AlertPolicyValidateProvider {
-	public String type;
-
-	public abstract GenericServiceAPIResponseEntity<String> validate();
-	
-	public abstract String PolicyType();
-	
-	public abstract List<Module> BindingModules();
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
index 9d77a58..d7b3232 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/config/ZKConfigBuilder.java
@@ -29,10 +29,10 @@ public class ZKConfigBuilder {
         ZKConfig zkConfig = new ZKConfig();
         zkConfig.zkQuorum = config.getString("zkConfig.zkQuorum");
         zkConfig.zkRoot = config.getString("zkConfig.zkRoot");
-        zkConfig.zkSessionTimeoutMs = config.getInt("zkConfig.zkSessionTimeoutMs");
-        zkConfig.connectionTimeoutMs = config.getInt("zkConfig.connectionTimeoutMs");
-        zkConfig.zkRetryTimes = config.getInt("zkConfig.zkRetryTimes");
-        zkConfig.zkRetryInterval = config.getInt("zkConfig.zkRetryInterval");
+        zkConfig.zkSessionTimeoutMs = config.hasPath("zkConfig.zkSessionTimeoutMs") ? config.getInt("zkConfig.zkSessionTimeoutMs")
: 10000;
+        zkConfig.connectionTimeoutMs = config.hasPath("zkConfig.connectionTimeoutMs") ? config.getInt("zkConfig.connectionTimeoutMs")
: 10000;
+        zkConfig.zkRetryTimes = config.hasPath("zkConfig.zkRetryTimes") ? config.getInt("zkConfig.zkRetryTimes")
: 3;
+        zkConfig.zkRetryInterval = config.hasPath("zkConfig.zkRetryInterval") ? config.getInt("zkConfig.zkRetryInterval")
: 3000;
         return zkConfig;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
index 1182e3f..226dd84 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/scheme/JsonStringStreamNameSelector.java
@@ -33,9 +33,9 @@ import org.slf4j.LoggerFactory;
  */
 public class JsonStringStreamNameSelector implements StreamNameSelector {
     private final static Logger LOG = LoggerFactory.getLogger(JsonStringStreamNameSelector.class);
-    private final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName";
-    private final static String FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY = "fieldNamesToInferStreamName";
-    private final static String STREAM_NAME_FORMAT = "streamNameFormat";
+    public final static String USER_PROVIDED_STREAM_NAME_PROPERTY = "userProvidedStreamName";
+    public final static String FIELD_NAMES_TO_INFER_STREAM_NAME_PROPERTY = "fieldNamesToInferStreamName";
+    public final static String STREAM_NAME_FORMAT = "streamNameFormat";
 
     private String userProvidedStreamName;
     private String[] fieldNamesToInferStreamName;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
index f54d5cd..99c1fed 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
@@ -332,9 +332,9 @@ public class CorrelationSpout extends BaseRichSpout implements SpoutSpecListener
             spoutConfig.zkPort = utils.getZkPort();
         }
         // transaction update interval
-        spoutConfig.stateUpdateIntervalMs = config.getLong("spout.stormKafkaStateUpdateIntervalMs");
+        spoutConfig.stateUpdateIntervalMs = config.hasPath("spout.stormKafkaStateUpdateIntervalMs")
? config.getInt("spout.stormKafkaStateUpdateIntervalMs") : 2000;
         // Kafka fetch size
-        spoutConfig.fetchSizeBytes = config.getInt("spout.stormKafkaFetchSizeBytes");
+        spoutConfig.fetchSizeBytes = config.hasPath("spout.stormKafkaFetchSizeBytes") ? config.getInt("spout.stormKafkaFetchSizeBytes")
: 1048586;
         // "startOffsetTime" is for test usage, prod should not use this
         if (config.hasPath("spout.stormKafkaStartOffsetTime")) {
             spoutConfig.startOffsetTime = config.getInt("spout.stormKafkaStartOffsetTime");

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
index 7030e45..dac2f07 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/application.conf
@@ -29,32 +29,17 @@
     "stormKafkaUseSameZkQuorumWithKafkaBroker": true,
     "stormKafkaTransactionZkQuorum": "",
     "stormKafkaTransactionZkPath": "/consumers",
-    "stormKafkaEagleConsumer": "eagle_consumer",
-    "stormKafkaStateUpdateIntervalMs": 2000,
-    "stormKafkaFetchSizeBytes": 1048586,
+    "stormKafkaEagleConsumer": "eagle_consumer"
   },
   "zkConfig" : {
     "zkQuorum" : "server.eagle.apache.org:2181",
-    "zkRoot" : "/alert",
-    "zkSessionTimeoutMs" : 10000,
-    "connectionTimeoutMs" : 10000,
-    "zkRetryTimes" : 3,
-    "zkRetryInterval" : 3000
-  },
-  "dynamicConfigSource" : {
-    "initDelayMillis": 3000,
-    "delayMillis" : 10000
+    "zkRoot" : "/alert"
   },
   "metadataService": {
     "context" : "/rest",
     "host" : "localhost",
     "port" : 9090
   },
-  "coordinatorService": {
-    "host": "localhost",
-    "port": 9090,
-    "context" : "/rest"
-  }
   "metric":{
     "sink": {
       // "kafka": {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
index aebf3b5..0cf5115 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/e2e/CoordinatorClient.java
@@ -41,9 +41,9 @@ public class CoordinatorClient implements Closeable {
     @SuppressWarnings("unused")
     private static final Logger LOG = LoggerFactory.getLogger(CoordinatorClient.class);
 
-    private static final String EAGLE_COORDINATOR_SERVICE_CONTEXT = "coordinatorService.context";
-    private static final String EAGLE_COORDINATOR_SERVICE_PORT = "coordinatorService.port";
-    private static final String EAGLE_COORDINATOR_SERVICE_HOST = "coordinatorService.host";
+    private static final String EAGLE_COORDINATOR_SERVICE_CONTEXT = "metadataService.context";
+    private static final String EAGLE_COORDINATOR_SERVICE_PORT = "metadataService.port";
+    private static final String EAGLE_COORDINATOR_SERVICE_HOST = "metadataService.host";
     private static final String COORDINATOR_SCHEDULE_API = "/coordinator/build";
 
     private String host;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-alert-parent/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/pom.xml b/eagle-core/eagle-alert-parent/pom.xml
index d5166cb..e2ba4d8 100644
--- a/eagle-core/eagle-alert-parent/pom.xml
+++ b/eagle-core/eagle-alert-parent/pom.xml
@@ -15,9 +15,7 @@
   ~ See the License for the specific language governing permissions and
   ~ limitations under the License.
   -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.eagle</groupId>
@@ -33,5 +31,6 @@
     <modules>
         <module>eagle-alert</module>
         <module>eagle-alert-service</module>
-    </modules>
-</project>
+    <module>eagle-alert-app</module>
+  </modules>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
index 52eb628..1a80b78 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
@@ -102,9 +102,9 @@ public class ApplicationContext implements Serializable, ApplicationLifecycle
{
                     datasource.setTopic(kafkaCfg.getTopicId());
                     datasource.setSchemeCls(JsonScheme.class.getCanonicalName());
                     Tuple2StreamMetadata tuple2Stream = new Tuple2StreamMetadata();
-                    Set<String> activeStreamNames = new HashSet<>();
-                    activeStreamNames.add(streamDesc.getSchema().getStreamId());
-                    tuple2Stream.setActiveStreamNames(activeStreamNames);
+                    Properties prop = new Properties();
+                    prop.put(JsonStringStreamNameSelector.USER_PROVIDED_STREAM_NAME_PROPERTY,
streamDesc.getStreamId());
+                    tuple2Stream.setStreamNameSelectorProp(prop);
                     tuple2Stream.setTimestampColumn("timestamp");
                     tuple2Stream.setStreamNameSelectorCls(JsonStringStreamNameSelector.class.getCanonicalName());
                     datasource.setCodec(tuple2Stream);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties b/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
index fb13ad5..d59ded6 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/resources/log4j.properties
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-log4j.rootLogger=DEBUG, stdout
+log4j.rootLogger=INFO, stdout
 
 # standard output
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/README.md b/eagle-security/eagle-security-hdfs-auditlog/README.md
new file mode 100644
index 0000000..8d2e97a
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/README.md
@@ -0,0 +1,191 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+Development in IDE
+
+## 1. Start eagle-server
+In IDE, configure the following main class and program arguments
+
+org.apache.eagle.server.ServerMain server src/main/resources/configuration.yml
+
+## 2. Start alert engine
+
+### 2.1 Create new site
+
+http://localhost:9090/rest/sites POST
+```
+{
+"siteId" : "testsite",
+"siteName" :"testsite",
+"description" : "test description",
+"context" : {}
+}
+```
+
+###n2.2 Create logic alert engine topology
+
+http://localhost:9090/rest/metadata/topologies POST
+```
+{
+   "name": "alertUnitTopology_1",
+   "numOfSpout": 1,
+   "numOfAlertBolt": 10,
+   "numOfGroupBolt": 4,
+   "spoutId": "alertEngineSpout",
+   "groupNodeIds": [
+      "streamRouterBolt0",
+      "streamRouterBolt1",
+      "streamRouterBolt2",
+      "streamRouterBolt3"
+   ],
+   "alertBoltIds": [
+      "alertBolt0",
+      "alertBolt1",
+      "alertBolt2",
+      "alertBolt3",
+      "alertBolt4",
+      "alertBolt5",
+      "alertBolt6",
+      "alertBolt7",
+      "alertBolt8",
+      "alertBolt9"
+   ],
+   "pubBoltId": "alertPublishBolt",
+   "spoutParallelism": 1,
+   "groupParallelism": 1,
+   "alertParallelism": 1
+}
+```
+
+### 2.3 Install alert engine application
+Please reference eagle-core/eagle-alert-parent/eagle-alert-app/src/main/resources/META-INF/providers/org.apache.eagle.alert.app.AlertUnitTopologyAppProvider.xml
for
+complete configuration.
+
+http://localhost:9090/rest/apps/install POST
+```
+{
+"siteId" : "testsite",
+"appType" : "AlertUnitTopologyApp",
+"mode" : "LOCAL",
+"configuration" : {
+  }
+}
+```
+
+### 2.4 Run alert engine
+Please use correct uuid
+
+http://localhost:9090/rest/apps/start POST
+```
+{
+"uuid": "dc61c4b8-f60d-4d95-bfd7-f6b07382a3f3",
+"appId": "AlertUnitTopologyApp´╝Źtestsite"
+}
+```
+
+## 3 Start Hdfs audit log monitoring application
+
+### 3.1 Install HdfsAuditLog app
+
+http://localhost:9090/rest/apps/install POST
+```
+{
+"siteId" : "testsite",
+"appType" : "HdfsAuditLogApplication",
+"mode" : "LOCAL",
+"configuration" : {
+  "dataSourceConfig.topic" :"hdfs_audit_log"}
+}
+```
+
+### 3.2 Start HdfsAuditLog app
+Please use correct uuid
+
+http://localhost:9090/rest/apps/start POST
+```
+{
+"uuid": "dc61c4b8-f60d-4d95-bfd7-f6b07382a3f3",
+"appId": "HdfsAuditLogApplication´╝Źtestsite"
+}
+```
+
+## 4 Check
+### 4.1 Check if alert data source is created
+http://localhost:9090/rest/metadata/datasources GET
+
+### 4.2 Check if alert stream is creatd
+http://localhost:9090/rest/metadata/streams GET
+
+## 5 Create alert policy and verify alert
+### 5.1 create one policy
+
+http://localhost:9090/rest/metadata/policies POST
+```
+{
+   "name": "hdfsPolicy",
+   "description": "hdfsPolicy",
+   "inputStreams": [
+      "hdfs_audit_log_enriched_stream"
+   ],
+   "outputStreams": [
+      "hdfs_audit_log_enriched_stream_out"
+   ],
+   "definition": {
+      "type": "siddhi",
+      "value": "from hdfs_audit_log_enriched_stream[user=='hadoop'] select * insert into
hdfs_audit_log_enriched_stream_out"
+   },
+   "partitionSpec": [
+      {
+         "streamId": "hdfs_audit_log_enriched_stream",
+         "type": "GROUPBY",
+         "columns" : [
+            "user"
+         ]
+      }
+   ],
+   "parallelismHint": 2
+}
+```
+
+### 5.2 Create alert publishment
+```
+{
+	"name":"hdfs_audit_log_enriched_stream_out",
+	"type":"org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher",
+	"policyIds": [
+		"hdfsPolicy"
+	],
+	"properties": {
+	  "subject":"alert when user is hadoop",
+	  "template":"",
+	  "sender": "eagle@apache.org",
+	  "recipients": "eagle@apache.org",
+	  "mail.smtp.host":"",
+	  "connection": "plaintext",
+	  "mail.smtp.port": "25"
+	},
+	"dedupIntervalMin" : "PT1M",
+	"serializer" : "org.apache.eagle.alert.engine.publisher.impl.StringEventSerializer"
+}
+```
+
+### 5.3 Send message and verify alert
+./kafka-console-producer.sh --topic hdfs_audit_log --broker-list sandbox.hortonworks.com:6667
+
+2015-04-24 12:51:31,798 INFO FSNamesystem.audit: allowed=true	ugi=hdfs (auth:SIMPLE)	ip=/10.0.2.15
cmd=getfileinfo	src=/apps/hbase/data	dst=null	perm=null	proto=rpc

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
index 2a8ff0f..2fb7274 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
@@ -155,21 +155,41 @@
             <timeseries>true</timeseries>
             <columns>
                 <column>
-                    <name>action</name>
+                    <name>src</name>
                     <type>string</type>
                 </column>
                 <column>
-                    <name>host</name>
+                    <name>dst</name>
                     <type>string</type>
                 </column>
                 <column>
-                    <name>status</name>
+                    <name>host</name>
                     <type>string</type>
                 </column>
                 <column>
                     <name>timestamp</name>
                     <type>long</type>
                 </column>
+                <column>
+                    <name>allowed</name>
+                    <type>bool</type>
+                </column>
+                <column>
+                    <name>user</name>
+                    <type>string</type>
+                </column>
+                <column>
+                    <name>cmd</name>
+                    <type>string</type>
+                </column>
+                <column>
+                    <name>sensitivityType</name>
+                    <type>string</type>
+                </column>
+                <column>
+                    <name>securityZone</name>
+                    <type>string</type>
+                </column>
             </columns>
         </stream>
     </streams>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt
new file mode 100644
index 0000000..c5f7c9b
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/scripts.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+./kafka-topics.sh --topic hdfs_audit_log --zookeeper localhost:2181 --create --replication-factor
1 --partitions 2
+
+./kafka-topics.sh --topic hdfs_audit_log --zookeeper localhost:2181 --create --replication-factor
1 --partitions 2
+
+./kafka-console-producer.sh --topic hdfs_audit_log --broker-list sandbox.hortonworks.com:6667
+
+./kafka-console-consumer.sh --topic hdfs_audit_log_parsed --zookeeper sandbox.hortonworks.com:2181
--from-beginning

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/000b7460/eagle-server/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml
index bc9a919..5e07a7d 100644
--- a/eagle-server/pom.xml
+++ b/eagle-server/pom.xml
@@ -164,6 +164,13 @@
             <artifactId>eagle-gc</artifactId>
             <version>${project.version}</version>
         </dependency>
+
+        <!-- Alert unit topology as an application -->
+        <dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-alert-app</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
     <build>
         <resources>



Mime
View raw message