eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject incubator-eagle git commit: EAGLE-338 fix topology-assembly build issue because of module name change fix topology-assembly build issue because of module name change
Date Sun, 12 Jun 2016 18:48:01 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/develop 8b646715e -> 7f6269cd9


EAGLE-338 fix topology-assembly build issue because of module name change
fix topology-assembly build issue because of module name change

Author: @yonzhang2012 <yonzhang2012@gmail.com>

Closes #236


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7f6269cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7f6269cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7f6269cd

Branch: refs/heads/develop
Commit: 7f6269cd913a2d31f45eba4e791b223011cfec0d
Parents: 8b64671
Author: yonzhang <yonzhang2012@gmail.com>
Authored: Sun Jun 12 11:50:36 2016 -0700
Committer: yonzhang <yonzhang2012@gmail.com>
Committed: Sun Jun 12 11:50:36 2016 -0700

----------------------------------------------------------------------
 .../service/HBaseSensitivityEntity.java         | 50 ++++++++++++++
 .../security/service/ISecurityMetadataDAO.java  | 27 ++++++++
 .../security/service/InMemMetadataDaoImpl.java  | 56 ++++++++++++++++
 .../security/service/MetadataDaoFactory.java    | 68 ++++++++++++++++++++
 .../apache/eagle/security/service/OpResult.java | 28 ++++++++
 .../service/SensitivityMetadataResource.java    | 27 ++++++++
 .../eagle-security-hbase-auditlog/README.md     | 64 +++++++++++++++++-
 .../hbase/HbaseAuditLogMonitoringMain.java      |  7 ++
 .../HbaseResourceSensitivityDataJoinBolt.java   | 64 ++++++++++--------
 .../HbaseResourceSensitivityPollingJob.java     | 22 ++-----
 .../src/main/resources/application.conf         |  9 ++-
 .../eagle-security-hdfs-authlog/README.md       | 52 +++++++++------
 eagle-topology-assembly/pom.xml                 |  9 +--
 eagle-webservice/pom.xml                        | 10 ---
 14 files changed, 411 insertions(+), 82 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
new file mode 100644
index 0000000..d46ef13
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.service;
+
+/**
+ * Since 6/10/16.
+ */
+public class HBaseSensitivityEntity {
+    private String site;
+    private String hbaseResource;
+    private String sensitivityType;
+
+    public String getSite() {
+        return site;
+    }
+
+    public void setSite(String site) {
+        this.site = site;
+    }
+
+    public String getHbaseResource() {
+        return hbaseResource;
+    }
+
+    public void setHbaseResource(String hbaseResource) {
+        this.hbaseResource = hbaseResource;
+    }
+
+    public String getSensitivityType() {
+        return sensitivityType;
+    }
+
+    public void setSensitivityType(String sensitivityType) {
+        this.sensitivityType = sensitivityType;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java
b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java
new file mode 100644
index 0000000..45e729e
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityMetadataDAO.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.service;
+
+import java.util.Collection;
+/**
+ * Since 6/10/16.
+ */
+public interface ISecurityMetadataDAO {
+    Collection<HBaseSensitivityEntity> listHBaseSensitivies();
+    OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
new file mode 100644
index 0000000..db38556
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.service;
+
+import com.typesafe.config.Config;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * In memory service for simple service start. Make all service API as
+ * synchronized.
+ * 
+ * @since Apr 11, 2016
+ *
+ */
+public class InMemMetadataDaoImpl implements ISecurityMetadataDAO {
+
+    private static final Logger LOG = LoggerFactory.getLogger(InMemMetadataDaoImpl.class);
+
+    private Map<Pair<String, String>, HBaseSensitivityEntity> hBaseSensitivityEntities
= new HashMap<>();
+
+    public InMemMetadataDaoImpl(Config config) {
+    }
+
+    @Override
+    public synchronized Collection<HBaseSensitivityEntity> listHBaseSensitivies() {
+        return hBaseSensitivityEntities.values();
+    }
+
+    @Override
+    public synchronized OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity>
h) {
+        for (HBaseSensitivityEntity e : h) {
+            Pair p = new ImmutablePair<>(e.getSite(), e.getHbaseResource());
+            hBaseSensitivityEntities.put(p, e);
+        }
+        return new OpResult();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataDaoFactory.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataDaoFactory.java
b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataDaoFactory.java
new file mode 100644
index 0000000..f17fd43
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataDaoFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.service;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Constructor;
+
+/**
+ * @since Apr 12, 2016
+ *
+ */
+public class MetadataDaoFactory {
+
+    private static final MetadataDaoFactory INSTANCE = new MetadataDaoFactory();
+    private static final Logger LOG = LoggerFactory.getLogger(MetadataDaoFactory.class);
+
+    private ISecurityMetadataDAO dao;
+
+    private MetadataDaoFactory() {
+        Config config = ConfigFactory.load();
+        Config datastoreConfig = config.getConfig("datastore");
+        if (datastoreConfig == null) {
+            LOG.warn("datastore is not configured, use in-memory store !!!");
+            dao = new InMemMetadataDaoImpl(datastoreConfig);
+        } else {
+            String clsName = datastoreConfig.getString("metadataDao");
+            Class<?> clz;
+            try {
+                clz = Thread.currentThread().getContextClassLoader().loadClass(clsName);
+                if (ISecurityMetadataDAO.class.isAssignableFrom(clz)) {
+                    Constructor<?> cotr = clz.getConstructor(Config.class);
+                    dao = (ISecurityMetadataDAO) cotr.newInstance(datastoreConfig);
+                } else {
+                    throw new Exception("metadataDao configuration need to be implementation
of IMetadataDao! ");
+                }
+            } catch (Exception e) {
+                LOG.error("error when initialize the dao, fall back to in memory mode!",
e);
+                dao = new InMemMetadataDaoImpl(datastoreConfig);
+            }
+        }
+    }
+
+    public static MetadataDaoFactory getInstance() {
+        return INSTANCE;
+    }
+
+    public ISecurityMetadataDAO getMetadataDao() {
+        return dao;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OpResult.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OpResult.java
b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OpResult.java
new file mode 100644
index 0000000..c264173
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/OpResult.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.service;
+
+/**
+ * @since Apr 11, 2016
+ *
+ */
+public class OpResult {
+
+    public int code = 200;
+    public String message = "";
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SensitivityMetadataResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SensitivityMetadataResource.java
b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SensitivityMetadataResource.java
new file mode 100644
index 0000000..05440fb
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SensitivityMetadataResource.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.service;
+
+import javax.ws.rs.Path;
+
+/**
+ * Since 6/10/16.
+ */
+@Path("/metadata/sensitivity")
+public class SensitivityMetadataResource {
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-hbase-auditlog/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/README.md b/eagle-security/eagle-security-hbase-auditlog/README.md
index e7e338c..b0a6f90 100644
--- a/eagle-security/eagle-security-hbase-auditlog/README.md
+++ b/eagle-security/eagle-security-hbase-auditlog/README.md
@@ -17,9 +17,69 @@ limitations under the License.
 {% endcomment %}
 -->
 
-#### create topic in sandbox
+Follow below steps to get Hdfs authorization logs monitoring running
+
+### Step 1: logstash: fetch log file to Kafka
+#### 1.1 create topic for raw log
 /usr/hdp/2.2.4.2-2/kafka/bin/kafka-topics.sh --create --topic sandbox_hbase_audit_log --partitions
2 --replication-factor 1 --zookeeper localhost:2181
+#### 1.2 consume raw log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-consumer.sh --topic sandbox_hbase_audit_log --zookeeper
sandbox.hortonworks.com:2181
+#### 1.3 create logstash config file: hbase-auditlog.conf
+download logstash 2.3.x
+~~~
+    input {
+        file {
+            type => "hbase-auditlog"
+            path => "/var/log/hbase/SecurityAuth.audit"
+            start_position => end
+            sincedb_path => "/var/log/logstash/hbase-auditlog-sincedb"
+        }
+    }
+
+    output {
+         if [type] == "hbase-auditlog" {
+              kafka {
+                  codec => plain {
+                      format => "%{message}"
+                  }
+                  bootstrap_servers => "sandbox.hortonworks.com:6667"
+                  topic_id => "sandbox_hbase_audit_log"
+                  acks => "0"
+                  timeout_ms => 10000
+                  retries => 3
+                  retry_backoff_ms => 100
+                  batch_size => 16384
+                  send_buffer_bytes => 131072
+                  client_id => "hbase-auditlog"
+              }
+              # stdout { codec => rubydebug }
+          }
+    }
+
+~~~
+#### 1.4 run logstash
+     bin/logstash -f hbase-auditlog.conf
+
+### Step 2: hbase-auditlog topology: parse log and persist to Kafka
+#### 2.1 create topic for normalized log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-topics.sh --create --topic sandbox_hbase_audit_log_parsed
--partitions 2 --replication-factor 1 --zookeeper localhost:2181
+#### 2.2 consume normalized log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-consumer.sh --topic sandbox_hbase_audit_log_parsed
--zookeeper sandbox.hortonworks.com:2181
+#### 2.3 run eagle webservice
+find eagle-webservice project, run it
+#### 2.4 run eagle-security-hdfs-authlog topology
+find org.apache.eagle.security.hbase.HbaseAuditLogMonitoringMain, run it
 
+### Step 3: alert engine: consume parsed log
+#### 3.1 run alert engine
+find org.apache.eagle.alert.engine.UnitTopologyMain, run it
 
-#### produce a message in sandbox
+
+### test to produce message
+##### produce raw log
 /usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-producer.sh --topic sandbox_hbase_audit_log --broker-list
sandbox.hortonworks.com:6667
+
+##### produce parsed log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-producer.sh --topic sandbox_hbase_audit_log_parsed
--broker-list sandbox.hortonworks.com:6667
+
+

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
index f231266..13ca214 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
@@ -30,12 +30,14 @@ import org.apache.eagle.security.topo.TopologySubmitter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import storm.kafka.*;
+import storm.kafka.bolt.KafkaBolt;
 
 public class HbaseAuditLogMonitoringMain {
     private static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogMonitoringMain.class);
     public final static String SPOUT_TASK_NUM = "topology.numOfSpoutTasks";
     public final static String PARSER_TASK_NUM = "topology.numOfParserTasks";
     public final static String JOIN_TASK_NUM = "topology.numOfJoinTasks";
+    public final static String SINK_TASK_NUM = "topology.numOfSinkTasks";
 
     public static void main(String[] args) throws Exception{
         System.setProperty("config.resource", "/application.conf");
@@ -49,6 +51,7 @@ public class HbaseAuditLogMonitoringMain {
         int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
         int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
         int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
+        int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
 
         builder.setSpout("ingest", spout, numOfSpoutTasks);
         BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
@@ -58,6 +61,10 @@ public class HbaseAuditLogMonitoringMain {
         BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
         joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
 
+        KafkaBolt kafkaBolt = new KafkaBolt();
+        BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", kafkaBolt, numOfSinkTasks);
+        kafkaBoltDeclarer.shuffleGrouping("joinBolt");
+
         StormTopology topology = builder.createTopology();
 
         TopologySubmitter.submit(topology, config);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
index 3a9416e..cf486d3 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
@@ -23,12 +23,14 @@ import backtype.storm.topology.OutputFieldsDeclarer;
 import backtype.storm.topology.base.BaseRichBolt;
 import backtype.storm.tuple.Fields;
 import backtype.storm.tuple.Tuple;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.typesafe.config.Config;
 import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
 import org.apache.eagle.security.util.ExternalDataCache;
 import org.apache.eagle.security.util.ExternalDataJoiner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import storm.kafka.bolt.mapper.FieldNameBasedTupleToKafkaMapper;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -60,42 +62,50 @@ public class HbaseResourceSensitivityDataJoinBolt extends BaseRichBolt
{
 
     @Override
     public void execute(Tuple input) {
-        @SuppressWarnings("unchecked")
-        Map<String, Object> event = (Map<String, Object>)input.getValue(0);
-        @SuppressWarnings("unchecked")
-        Map<String, HbaseResourceSensitivityAPIEntity> map =
-                (Map<String, HbaseResourceSensitivityAPIEntity>) ExternalDataCache
-                        .getInstance()
-                        .getJobResult(HbaseResourceSensitivityPollingJob.class);
-        LOG.info(">>>> event: " + event + " >>>> map: " + map);
+        try {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> event = (Map<String, Object>) input.getValue(0);
+            @SuppressWarnings("unchecked")
+            Map<String, HbaseResourceSensitivityAPIEntity> map =
+                    (Map<String, HbaseResourceSensitivityAPIEntity>) ExternalDataCache
+                            .getInstance()
+                            .getJobResult(HbaseResourceSensitivityPollingJob.class);
+            LOG.info(">>>> event: " + event + " >>>> map: " + map);
 
-        String resource = (String)event.get("scope");
+            String resource = (String) event.get("scope");
 
-        HbaseResourceSensitivityAPIEntity sensitivityEntity = null;
+            HbaseResourceSensitivityAPIEntity sensitivityEntity = null;
 
-        if (map != null && resource != "") {
-            for (String key : map.keySet()) {
-                Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
-                if(pattern.matcher(resource).find()) {
-                    sensitivityEntity = map.get(key);
-                    break;
+            if (map != null && resource != "") {
+                for (String key : map.keySet()) {
+                    Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
+                    if (pattern.matcher(resource).find()) {
+                        sensitivityEntity = map.get(key);
+                        break;
+                    }
                 }
             }
+            Map<String, Object> newEvent = new TreeMap<String, Object>(event);
+            newEvent.put("sensitivityType", sensitivityEntity == null ?
+                    "NA" : sensitivityEntity.getSensitivityType());
+            newEvent.put("scope", resource);
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("After hbase resource sensitivity lookup: " + newEvent);
+            }
+            LOG.info("After hbase resource sensitivity lookup: " + newEvent);
+            // push to Kafka sink
+            ObjectMapper mapper = new ObjectMapper();
+            String msg = mapper.writeValueAsString(map);
+            collector.emit(Arrays.asList(newEvent.get("user"), msg));
+        }catch(Exception ex){
+            LOG.error("error joining data, ignore it", ex);
+        }finally {
+            collector.ack(input);
         }
-        Map<String, Object> newEvent = new TreeMap<String, Object>(event);
-        newEvent.put("sensitivityType", sensitivityEntity  == null ?
-                "NA" : sensitivityEntity.getSensitivityType());
-        newEvent.put("scope", resource);
-        if(LOG.isDebugEnabled()) {
-            LOG.debug("After hbase resource sensitivity lookup: " + newEvent);
-        }
-        LOG.info("After hbase resource sensitivity lookup: " + newEvent);
-        collector.emit(Arrays.asList(newEvent.get("user"), newEvent));
     }
 
     @Override
     public void declareOutputFields(OutputFieldsDeclarer declarer) {
-        declarer.declare(new Fields("f1"));
-
+        declarer.declare(new Fields(FieldNameBasedTupleToKafkaMapper.BOLT_KEY, FieldNameBasedTupleToKafkaMapper.BOLT_MESSAGE));
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
index 6d6c77a..9ca0701 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
@@ -20,6 +20,9 @@ package org.apache.eagle.security.hbase;
 import com.google.common.base.Function;
 import com.google.common.collect.Maps;
 import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
+import org.apache.eagle.security.service.HBaseSensitivityEntity;
+import org.apache.eagle.security.service.ISecurityMetadataDAO;
+import org.apache.eagle.security.service.MetadataDaoFactory;
 import org.apache.eagle.security.util.AbstractResourceSensitivityPollingJob;
 import org.apache.eagle.security.util.ExternalDataCache;
 import org.quartz.Job;
@@ -29,6 +32,7 @@ import org.quartz.JobExecutionException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -40,21 +44,9 @@ public class HbaseResourceSensitivityPollingJob extends AbstractResourceSensitiv
             throws JobExecutionException {
         JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
         try {
-            List<HbaseResourceSensitivityAPIEntity>
-            hbaseResourceSensitivity = load(jobDataMap, "HbaseResourceSensitivityService");
-            if(hbaseResourceSensitivity == null) {
-            	LOG.warn("Hbase resource sensitivity information is empty");
-            	return;
-            }
-            Map<String, HbaseResourceSensitivityAPIEntity> map = Maps.uniqueIndex(
-            		hbaseResourceSensitivity,
-            		new Function<HbaseResourceSensitivityAPIEntity, String>() {
-            			@Override
-            			public String apply(HbaseResourceSensitivityAPIEntity input) {
-            				return input.getTags().get("hbaseResource");
-            			}
-            		});
-            ExternalDataCache.getInstance().setJobResult(getClass(), map);
+            ISecurityMetadataDAO dao = MetadataDaoFactory.getInstance().getMetadataDao();
+            Collection<HBaseSensitivityEntity> sensitivityEntities = dao.listHBaseSensitivies();
+            ExternalDataCache.getInstance().setJobResult(getClass(), sensitivityEntities);
         } catch(Exception ex) {
         	LOG.error("Fail to load hbase resource sensitivity data", ex);
         }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
index 4eaa58a..bd38c83 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
@@ -20,6 +20,7 @@
     "numOfSpoutTasks" : 2,
     "numOfParserTasks" : 2,
     "numOfJoinTasks" : 2,
+    "numOfSinkTasks" : 2,
     "name" : "sandbox-hbaseAuditLog-topology",
   },
   "dataSourceConfig": {
@@ -38,9 +39,15 @@
     "dataJoinPollIntervalSec" : 30,
     "eagleService": {
       "host": "localhost",
-      "port": 9098
+      "port": 58080
       "username": "admin",
       "password": "secret"
     }
+  },
+  "dataSinkConfig": {
+    "topic" : "sandbox_hbase_audit_log_parsed",
+    "brokerList" : "sandbox.hortonworks.com:6667",
+    "serializerClass" : "kafka.serializer.StringEncoder",
+    "keySerializerClass" : "kafka.serializer.StringEncoder"
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-security/eagle-security-hdfs-authlog/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-authlog/README.md b/eagle-security/eagle-security-hdfs-authlog/README.md
index e1a1971..328b72b 100644
--- a/eagle-security/eagle-security-hdfs-authlog/README.md
+++ b/eagle-security/eagle-security-hdfs-authlog/README.md
@@ -17,34 +17,27 @@ limitations under the License.
 {% endcomment %}
 -->
 
-#### create topic in sandbox
-/usr/hdp/2.2.4.2-2/kafka/bin/kafka-topics.sh --create --topic sandbox_hdfs_auth_log --partitions
2 --replication-factor 1 --zookeeper localhost:2181
-/usr/hdp/2.2.4.2-2/kafka/bin/kafka-topics.sh --create --topic sandbox_hdfs_auth_log_parsed
--partitions 2 --replication-factor 1 --zookeeper localhost:2181
-
-#### produce a message
-/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-producer.sh --topic sandbox_hdfs_auth_log --broker-list
sandbox.hortonworks.com:6667
+Follow below steps to get Hdfs authorization logs monitoring running
 
-
-#### consume parsed hdfs authorization log
+### Step 1: logstash: fetch log file to Kafka
+#### 1.1 create topic for raw log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-topics.sh --create --topic sandbox_hdfs_auth_log --partitions
2 --replication-factor 1 --zookeeper localhost:2181
+#### 1.2 consume raw log
 /usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-consumer.sh --topic sandbox_hdfs_auth_log --zookeeper
sandbox.hortonworks.com:2181
-
-
-#### run logstash in sandbox to fetch Hdfs authorization log
-
-##### step 1: create logstash config file: hdfs-authlog.conf
-
+#### 1.3 create logstash config file: hdfs-authlog.conf
+download logstash 2.3.x
 ~~~
     input {
         file {
-            type => "hdfs-auth"
+            type => "hdfs-authlog"
             path => "/var/log/hadoop/hdfs/SecurityAuth.audit"
             start_position => end
-            sincedb_path => "/var/log/logstash/sincedb"
+            sincedb_path => "/var/log/logstash/hdfs-authlog-sincedb"
         }
     }
 
     output {
-         if [type] == "hdfs-auth" {
+         if [type] == "hdfs-authlog" {
               kafka {
                   codec => plain {
                       format => "%{message}"
@@ -57,15 +50,34 @@ limitations under the License.
                   retry_backoff_ms => 100
                   batch_size => 16384
                   send_buffer_bytes => 131072
-                  client_id => "hdfs-auth"
+                  client_id => "hdfs-authlog"
               }
               # stdout { codec => rubydebug }
           }
     }
 
 ~~~
+#### 1.4 run logstash
+     bin/logstash -f hdfs-authlog.conf
 
-##### step 2: run logstash
+### Step 2: hdfs-authlog topology: parse log and persist to Kafka
+#### 2.1 create topic for normalized log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-topics.sh --create --topic sandbox_hdfs_auth_log_parsed
--partitions 2 --replication-factor 1 --zookeeper localhost:2181
+#### 2.2 consume normalized log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-consumer.sh --topic sandbox_hdfs_auth_log_parsed
--zookeeper sandbox.hortonworks.com:2181
+#### 2.3 run eagle webservice
+find eagle-webservice project, run it
+#### 2.4 run eagle-security-hdfs-authlog topology
+find org.apache.eagle.security.securitylog.HdfsAuthLogMonitoringMain, run it
 
-bin/logstash -f hdfs-authlog.conf
+### Step 3: alert engine: consume parsed log
+#### 3.1 run alert engine
+find org.apache.eagle.alert.engine.UnitTopologyMain, run it
+
+
+### test to produce message
+##### produce raw log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-producer.sh --topic sandbox_hdfs_auth_log --broker-list
sandbox.hortonworks.com:6667
 
+##### produce parsed log
+/usr/hdp/2.2.4.2-2/kafka/bin/kafka-console-producer.sh --topic sandbox_hdfs_auth_log_parsed
--broker-list sandbox.hortonworks.com:6667

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-topology-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml
index e40a234..d454416 100644
--- a/eagle-topology-assembly/pom.xml
+++ b/eagle-topology-assembly/pom.xml
@@ -35,7 +35,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-security-hdfs-securitylog</artifactId>
+            <artifactId>eagle-security-hdfs-authlog</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
@@ -45,17 +45,12 @@
         </dependency>
         <dependency>
             <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-security-userprofile-detection</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.eagle</groupId>
             <artifactId>eagle-security-oozie-auditlog</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-security-hbase-securitylog</artifactId>
+            <artifactId>eagle-security-hbase-auditlog</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7f6269cd/eagle-webservice/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-webservice/pom.xml b/eagle-webservice/pom.xml
index 1065506..9b00f66 100644
--- a/eagle-webservice/pom.xml
+++ b/eagle-webservice/pom.xml
@@ -210,22 +210,12 @@
 
 		<!-- eagle user profile common dependency -->
 		<dependency>
-			<groupId>org.apache.eagle</groupId>
-			<artifactId>eagle-security-userprofile-common</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
 			<groupId>org.apache.tomcat</groupId>
 			<artifactId>tomcat-catalina</artifactId>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.eagle</groupId>
-			<artifactId>eagle-machinelearning-base</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.eagle</groupId>
 			<artifactId>eagle-alert-base</artifactId>
 			<version>${project.version}</version>
 			<exclusions>


Mime
View raw message