eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject [2/4] incubator-eagle git commit: EAGLE-331 ingestion+alert engine preview ingestion + alert engine preview
Date Thu, 09 Jun 2016 23:18:48 GMT
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java
new file mode 100644
index 0000000..79eadd1
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Since 6/7/16.
+ */
+public class HBaseAuditLogParserBolt extends BaseRichBolt {
+    private static Logger LOG = LoggerFactory.getLogger(HBaseAuditLogParserBolt.class);
+    private OutputCollector collector;
+
+    @Override
+    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+        this.collector = collector;
+    }
+
+    @Override
+    public void execute(Tuple input) {
+        String logLine = new String(input.getString(0));
+
+        HbaseAuditLogParser parser = new HbaseAuditLogParser();
+        try{
+            HbaseAuditLogObject entity = parser.parse(logLine);
+            Map<String, Object> map = new TreeMap<String, Object>();
+            map.put("action", entity.action);
+            map.put("host", entity.host);
+            map.put("status", entity.status);
+            map.put("request", entity.request);
+            map.put("scope", entity.scope);
+            map.put("user", entity.user);
+            map.put("timestamp", entity.timestamp);
+            collector.emit(Arrays.asList(map));
+        }catch(Exception ex){
+            LOG.error("Failing parse and ignore audit log {} ", logLine, ex);
+        }finally {
+            collector.ack(input);
+        }
+    }
+
+    @Override
+    public void declareOutputFields(OutputFieldsDeclarer declarer) {
+        declarer.declare(new Fields("f1"));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogKafkaDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogKafkaDeserializer.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogKafkaDeserializer.java
new file mode 100644
index 0000000..5b37519
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogKafkaDeserializer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+
+import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+
+public class HbaseAuditLogKafkaDeserializer implements SpoutKafkaMessageDeserializer {
+    private static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogKafkaDeserializer.class);
+    private Properties props;
+
+    public HbaseAuditLogKafkaDeserializer(Properties props){
+        this.props = props;
+    }
+
+    @Override
+    public Object deserialize(byte[] arg0) {
+        String logLine = new String(arg0);
+
+        HbaseAuditLogParser parser = new HbaseAuditLogParser();
+        try{
+            HbaseAuditLogObject entity = parser.parse(logLine);
+            if(entity == null) return null;
+
+            Map<String, Object> map = new TreeMap<String, Object>();
+            map.put("action", entity.action);
+            map.put("host", entity.host);
+            map.put("status", entity.status);
+            map.put("request", entity.request);
+            map.put("scope", entity.scope);
+            map.put("user", entity.user);
+            map.put("timestamp", entity.timestamp);
+            return map;
+        }catch(Exception ex){
+            LOG.error("Failing parse audit log:" + logLine, ex);
+            return null;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
new file mode 100644
index 0000000..f231266
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringMain.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.hbase;
+
+import backtype.storm.generated.StormTopology;
+import backtype.storm.topology.BoltDeclarer;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.topology.TopologyBuilder;
+import backtype.storm.tuple.Fields;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.security.topo.NewKafkaSourcedSpoutProvider;
+import org.apache.eagle.security.topo.TopologySubmitter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import storm.kafka.*;
+
+public class HbaseAuditLogMonitoringMain {
+    private static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogMonitoringMain.class);
+    public final static String SPOUT_TASK_NUM = "topology.numOfSpoutTasks";
+    public final static String PARSER_TASK_NUM = "topology.numOfParserTasks";
+    public final static String JOIN_TASK_NUM = "topology.numOfJoinTasks";
+
+    public static void main(String[] args) throws Exception{
+        System.setProperty("config.resource", "/application.conf");
+        Config config = ConfigFactory.load();
+        NewKafkaSourcedSpoutProvider provider = new NewKafkaSourcedSpoutProvider();
+        IRichSpout spout = provider.getSpout(config);
+
+        HBaseAuditLogParserBolt bolt = new HBaseAuditLogParserBolt();
+        TopologyBuilder builder = new TopologyBuilder();
+
+        int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
+        int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
+        int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
+
+        builder.setSpout("ingest", spout, numOfSpoutTasks);
+        BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
+        boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));
+
+        HbaseResourceSensitivityDataJoinBolt joinBolt = new HbaseResourceSensitivityDataJoinBolt(config);
+        BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
+        joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
+
+        StormTopology topology = builder.createTopology();
+
+        TopologySubmitter.submit(topology, config);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java
new file mode 100644
index 0000000..ed9367d
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.hbase;
+
+
+public class HbaseAuditLogObject {
+    public long timestamp;
+    public String user = "";
+    public String scope = "";
+    public String action = "";
+    public String host = "";
+    public String request = "";
+    public String status = "";
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java
new file mode 100644
index 0000000..f9b74e6
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+import java.io.Serializable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.security.util.LogParseUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class HbaseAuditLogParser implements Serializable {
+    private final static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogParser.class);
+
+    private final static int LOGDATE_INDEX = 1;
+    private final static int LOGLEVEL_INDEX = 2;
+    private final static int LOGATTRS_INDEX = 3;
+    private final static String ALLOWED = "allowed";
+    private final static String DENIED = "denied";
+    private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*)");
+    private final static Pattern loggerContextPattern = Pattern.compile("\\w+:\\s*\\(user=(.*),\\s*scope=(.*),\\s*family=(.*),\\s*action=(.*)\\)");
+    private final static Pattern allowedPattern = Pattern.compile(ALLOWED);
+
+
+    public HbaseAuditLogObject parse(String logLine) {
+        if(logLine == null || logLine.isEmpty()) return null;
+
+        HbaseAuditLogObject ret = new HbaseAuditLogObject();
+        String timestamp = "";
+        String user = "";
+        String scope = "";
+        String action = "";
+        String ip = "";
+        String request = "";
+        String family = "";
+        String context = "";
+
+        Matcher loggerMatcher = loggerPattern.matcher(logLine);
+        if(loggerMatcher.find()) {
+            try {
+                timestamp = loggerMatcher.group(LOGDATE_INDEX);
+                String [] attrs = loggerMatcher.group(LOGATTRS_INDEX).split(";");
+                ret.status = allowedPattern.matcher(attrs[0]).find() ? ALLOWED : DENIED;
+                try {
+                    ip = attrs[2].split(":")[1].trim();
+                } catch (Exception e) {
+                    ip = "";
+                }
+                try {
+                    request = attrs[3].split(":")[1].trim();
+                } catch (Exception e) {
+                    request = "";
+                }
+                try {
+                    context = attrs[4].trim();
+                } catch (Exception e) {
+                    context = "";
+                }
+                Matcher contextMatcher = loggerContextPattern.matcher(context);
+                if(contextMatcher.find()) {
+                    user = contextMatcher.group(1);
+                    scope = contextMatcher.group(2);
+                    family = contextMatcher.group(3);
+                    action = contextMatcher.group(4);
+                }
+                if(StringUtils.isNotEmpty(family)) {
+                    if(!scope.contains(":")) scope = "default:" + scope;
+                    scope = String.format("%s:%s", scope, family);
+                }
+                if(StringUtils.isNotEmpty(ip)) {
+                    ret.host = ip.substring(1);
+                }
+                ret.timestamp = DateTimeUtil.humanDateToMilliseconds(timestamp);
+                ret.scope = scope;
+                ret.action = action;
+                ret.user = LogParseUtil.parseUserFromUGI(user);
+                ret.request = request;
+                return ret;
+            } catch(Exception e) {
+                LOG.error("Got exception when parsing audit log:" + logLine + ", exception:" + e.getMessage(), e);
+            }
+        }
+        return null;
+    }
+}
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
new file mode 100644
index 0000000..3a9416e
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
+import org.apache.eagle.security.util.ExternalDataCache;
+import org.apache.eagle.security.util.ExternalDataJoiner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
+
+public class HbaseResourceSensitivityDataJoinBolt extends BaseRichBolt {
+    private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityDataJoinBolt.class);
+    private Config config;
+    private OutputCollector collector;
+
+    public HbaseResourceSensitivityDataJoinBolt(Config config){
+        this.config = config;
+    }
+
+    @Override
+    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+        this.collector = collector;
+        // start hive resource data polling
+        try {
+            ExternalDataJoiner joiner = new ExternalDataJoiner(
+                    HbaseResourceSensitivityPollingJob.class, config, context.getThisComponentId() + "." + context.getThisTaskIndex());
+            joiner.start();
+        } catch(Exception ex){
+            LOG.error("Fail to bring up quartz scheduler.", ex);
+            throw new IllegalStateException(ex);
+        }
+    }
+
+    @Override
+    public void execute(Tuple input) {
+        @SuppressWarnings("unchecked")
+        Map<String, Object> event = (Map<String, Object>)input.getValue(0);
+        @SuppressWarnings("unchecked")
+        Map<String, HbaseResourceSensitivityAPIEntity> map =
+                (Map<String, HbaseResourceSensitivityAPIEntity>) ExternalDataCache
+                        .getInstance()
+                        .getJobResult(HbaseResourceSensitivityPollingJob.class);
+        LOG.info(">>>> event: " + event + " >>>> map: " + map);
+
+        String resource = (String)event.get("scope");
+
+        HbaseResourceSensitivityAPIEntity sensitivityEntity = null;
+
+        if (map != null && resource != "") {
+            for (String key : map.keySet()) {
+                Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
+                if(pattern.matcher(resource).find()) {
+                    sensitivityEntity = map.get(key);
+                    break;
+                }
+            }
+        }
+        Map<String, Object> newEvent = new TreeMap<String, Object>(event);
+        newEvent.put("sensitivityType", sensitivityEntity  == null ?
+                "NA" : sensitivityEntity.getSensitivityType());
+        newEvent.put("scope", resource);
+        if(LOG.isDebugEnabled()) {
+            LOG.debug("After hbase resource sensitivity lookup: " + newEvent);
+        }
+        LOG.info("After hbase resource sensitivity lookup: " + newEvent);
+        collector.emit(Arrays.asList(newEvent.get("user"), newEvent));
+    }
+
+    @Override
+    public void declareOutputFields(OutputFieldsDeclarer declarer) {
+        declarer.declare(new Fields("f1"));
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
new file mode 100644
index 0000000..6d6c77a
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
+import org.apache.eagle.security.util.AbstractResourceSensitivityPollingJob;
+import org.apache.eagle.security.util.ExternalDataCache;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class HbaseResourceSensitivityPollingJob extends AbstractResourceSensitivityPollingJob implements Job {
+    private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityPollingJob.class);
+
+    @Override
+    public void execute(JobExecutionContext context)
+            throws JobExecutionException {
+        JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
+        try {
+            List<HbaseResourceSensitivityAPIEntity>
+            hbaseResourceSensitivity = load(jobDataMap, "HbaseResourceSensitivityService");
+            if(hbaseResourceSensitivity == null) {
+            	LOG.warn("Hbase resource sensitivity information is empty");
+            	return;
+            }
+            Map<String, HbaseResourceSensitivityAPIEntity> map = Maps.uniqueIndex(
+            		hbaseResourceSensitivity,
+            		new Function<HbaseResourceSensitivityAPIEntity, String>() {
+            			@Override
+            			public String apply(HbaseResourceSensitivityAPIEntity input) {
+            				return input.getTags().get("hbaseResource");
+            			}
+            		});
+            ExternalDataCache.getInstance().setJobResult(getClass(), map);
+        } catch(Exception ex) {
+        	LOG.error("Fail to load hbase resource sensitivity data", ex);
+        }
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
new file mode 100644
index 0000000..4eaa58a
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "topology" : {
+    "localMode" : true,
+    "numOfTotalWorkers" : 2,
+    "numOfSpoutTasks" : 2,
+    "numOfParserTasks" : 2,
+    "numOfJoinTasks" : 2,
+    "name" : "sandbox-hbaseAuditLog-topology",
+  },
+  "dataSourceConfig": {
+    "topic" : "sandbox_hbase_audit_log",
+    "zkConnection" : "sandbox.hortonworks.com:2181",
+    "zkConnectionTimeoutMS" : 15000,
+    "fetchSize" : 1048586,
+    "transactionZKServers" : "sandbox.hortonworks.com",
+    "transactionZKPort" : 2181,
+    "transactionZKRoot" : "/consumers",
+    "consumerGroupId" : "eagle.hbaseaudit.consumer",
+    "transactionStateUpdateMS" : 2000,
+    "schemeCls" : "storm.kafka.StringScheme"
+  },
+  "eagleProps" : {
+    "dataJoinPollIntervalSec" : 30,
+    "eagleService": {
+      "host": "localhost",
+      "port": 9098
+      "username": "admin",
+      "password": "secret"
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf.bak
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf.bak b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf.bak
new file mode 100644
index 0000000..5c44574
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/application.conf.bak
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "envContextConfig" : {
+    "env" : "storm",
+    "mode" : "local",
+    "topologyName" : "sandbox-hbaseSecurityLog-topology",
+    "stormConfigFile" : "security-auditlog-storm.yaml",
+    "parallelismConfig" : {
+      "kafkaMsgConsumer" : 1,
+      "hbaseSecurityLogAlertExecutor*" : 1
+    }
+  },
+  "dataSourceConfig": {
+    "topic" : "sandbox_hbase_security_log",
+    "zkConnection" : "sandbox.hortonworks.com:2181",
+    "zkConnectionTimeoutMS" : 15000,
+    "consumerGroupId" : "EagleConsumer",
+    "fetchSize" : 1048586,
+    "deserializerClass" : "org.apache.eagle.security.hbase.HbaseAuditLogKafkaDeserializer",
+    "transactionZKServers" : "sandbox.hortonworks.com",
+    "transactionZKPort" : 2181,
+    "transactionZKRoot" : "/consumers",
+    "consumerGroupId" : "eagle.hbasesecurity.consumer",
+    "transactionStateUpdateMS" : 2000
+  },
+  "alertExecutorConfigs" : {
+     "hbaseSecurityLogAlertExecutor" : {
+       "parallelism" : 1,
+       "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner"
+       "needValidation" : "true"
+     }
+  },
+  "eagleProps" : {
+    "site" : "sandbox",
+    "application": "hbaseSecurityLog",
+    "dataJoinPollIntervalSec" : 30,
+    "mailHost" : "mailHost.com",
+    "mailSmtpPort":"25",
+    "mailDebug" : "true",
+    "eagleService": {
+      "host": "localhost",
+      "port": 9098
+      "username": "admin",
+      "password": "secret"
+    }
+  },
+  "dynamicConfigSource" : {
+    "enabled" : true,
+    "initDelayMillis" : 0,
+    "delayMillis" : 30000
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/log4j.properties b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/log4j.properties
new file mode 100644
index 0000000..d59ded6
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
new file mode 100644
index 0000000..467cc2f
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TestHbaseAuditLogParser {
+    HbaseAuditLogParser parser = new HbaseAuditLogParser();
+
+    @Test
+    public void test() throws Exception {
+        String log = "2015-08-11 13:31:03,729 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Table permission granted; remote address: /127.0.0.1; request: get; context: (user=eagle,scope=hbase:namespace,family=info, action=READ)";
+        HbaseAuditLogObject obj = parser.parse(log);
+        Assert.assertEquals(obj.action, "READ");
+        Assert.assertEquals(obj.host, "127.0.0.1");
+        Assert.assertEquals(obj.scope, "hbase:namespace:info");
+    }
+
+    @Test
+    public void test2() throws Exception {
+        String log = "2015-08-04 12:29:03,073 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Global check allowed; remote address: ; request: preOpen; context: (user=eagle, scope=GLOBAL, family=, action=ADMIN)";
+        HbaseAuditLogObject obj = parser.parse(log);
+        Assert.assertEquals(obj.action, "ADMIN");
+        Assert.assertEquals(obj.host, "");
+        Assert.assertEquals(obj.scope, "GLOBAL");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java
new file mode 100644
index 0000000..6abc966
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.hbase;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigParseOptions;
+import com.typesafe.config.ConfigSyntax;
+import org.apache.eagle.common.config.EagleConfigConstants;
+import org.junit.Test;
+
+
+public class TestHbaseAuditLogProcessTopology {
+    @Test
+    public void test() throws Exception {
+        //Config baseConfig = ConfigFactory.load("eagle-scheduler.conf");
+        ConfigParseOptions options = ConfigParseOptions.defaults()
+                .setSyntax(ConfigSyntax.PROPERTIES)
+                .setAllowMissing(false);
+        String topoConfigStr = "web.hbase.zookeeper.property.clientPort=2181\nweb.hbase.zookeeper.quorum=sandbox.hortonworks.com\n\napp.envContextConfig.env=storm\napp.envContextConfig.mode=local\napp.dataSourceConfig.topic=sandbox_hbase_security_log\napp.dataSourceConfig.zkConnection=sandbox.hortonworks.com:2181\napp.dataSourceConfig.zkConnectionTimeoutMS=15000\napp.dataSourceConfig.brokerZkPath=/brokers\napp.dataSourceConfig.fetchSize=1048586\napp.dataSourceConfig.transactionZKServers=sandbox.hortonworks.com\napp.dataSourceConfig.transactionZKPort=2181\napp.dataSourceConfig.transactionZKRoot=/consumers\napp.dataSourceConfig.consumerGroupId=eagle.hbasesecurity.consumer\napp.dataSourceConfig.transactionStateUpdateMS=2000\napp.dataSourceConfig.deserializerClass=org.apache.eagle.security.hbase.HbaseAuditLogKafkaDeserializer\napp.eagleProps.site=sandbox\napp.eagleProps.application=hbaseSecurityLog\napp.eagleProps.dataJoinPollIntervalSec=30\napp.eagleProps.mailHost=mailHost.com\napp.eag
 leProps.mailSmtpPort=25\napp.eagleProps.mailDebug=true\napp.eagleProps.eagleService.host=localhost\napp.eagleProps.eagleService.port=9098\napp.eagleProps.eagleService.username=admin\napp.eagleProps.eagleService.password=secret";
+
+        Config topoConfig = ConfigFactory.parseString(topoConfigStr, options);
+        Config conf = topoConfig.getConfig(EagleConfigConstants.APP_CONFIG);
+
+        HbaseAuditLogMonitoringMain topology = new HbaseAuditLogMonitoringMain();
+        //topology.submit("", conf);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/resources/application.conf b/eagle-security/eagle-security-hbase-auditlog/src/test/resources/application.conf
new file mode 100644
index 0000000..d2c53c7
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/test/resources/application.conf
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "envContextConfig" : {
+    "env" : "storm",
+    "mode" : "local",
+    "topologyName" : "sandbox-hbaseSecurityLog-topology",
+    "stormConfigFile" : "security-auditlog-storm.yaml",
+    "parallelismConfig" : {
+      "kafkaMsgConsumer" : 1,
+      "hbaseSecurityLogAlertExecutor*" : 1
+    }
+  },
+  "dataSourceConfig": {
+    "topic" : "sandbox_hbase_security_log",
+    "zkConnection" : "sandbox.hortonworks.com:2181",
+    "zkConnectionTimeoutMS" : 15000,
+    "consumerGroupId" : "EagleConsumer",
+    "fetchSize" : 1048586,
+    "deserializerClass" : "org.apache.eagle.security.hbase.HbaseAuditLogKafkaDeserializer",
+    "transactionZKServers" : "sandbox.hortonworks.com",
+    "transactionZKPort" : 2181,
+    "transactionZKRoot" : "/consumers",
+    "consumerGroupId" : "eagle.hbasesecurity.consumer",
+    "transactionStateUpdateMS" : 2000
+  },
+  "alertExecutorConfigs" : {
+    "hbaseSecurityLogAlertExecutor" : {
+      "parallelism" : 1,
+      "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner"
+      "needValidation" : "true"
+    }
+  },
+  "eagleProps" : {
+    "site" : "sandbox",
+    "application": "hbaseSecurityLog",
+    "dataJoinPollIntervalSec" : 30,
+    "mailHost" : "smtp.office365.com",
+    "mailSmtpPort":"587",
+    "mailSmtpAuth" : "true",
+    "mailSmtpUser" : "username",
+    "mailSmtpPassword" : "password",
+    #"mailSmtpSslEnable" : "true",
+    "mailSmtpTlsEnable" : "true",
+    "mailDebug" : "true",
+    "eagleService": {
+      "host": "localhost",
+      "port": 9098
+      "username": "admin",
+      "password": "secret"
+    }
+  },
+  "dynamicConfigSource" : {
+    "enabled" : true,
+    "initDelayMillis" : 0,
+    "delayMillis" : 30000
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/resources/log4j.properties b/eagle-security/eagle-security-hbase-auditlog/src/test/resources/log4j.properties
new file mode 100644
index 0000000..25331ab
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/test/resources/log4j.properties
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, stdout
+
+ eagle.log.dir=../logs
+ eagle.log.file=eagle.log
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
+
+# Daily Rolling File Appender
+ log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+ log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
+ log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+## 30-day backup
+# log4j.appender.DRFA.MaxBackupIndex=30
+ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-auditlog/src/test/resources/securityAuditLog
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/resources/securityAuditLog b/eagle-security/eagle-security-hbase-auditlog/src/test/resources/securityAuditLog
new file mode 100644
index 0000000..8c7ea52
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/test/resources/securityAuditLog
@@ -0,0 +1,25 @@
+2015-11-06 13:12:51,767 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user hbase; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=hbase, scope=hbase:meta, family=info, action=READ)
+2015-11-06 13:12:51,767 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user hbase; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=hbase, scope=hbase:meta, family=info, action=READ)
+2015-11-06 13:13:59,667 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-06 13:13:59,667 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-06 13:14:00,137 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access denied for user root; reason: Global check failed; remote address: /192.168.56.101; request: createTable; context: (user=root, scope=alertdef, family=f, action=CREATE)
+2015-11-06 13:14:00,137 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access denied for user root; reason: Global check failed; remote address: /192.168.56.101; request: createTable; context: (user=root, scope=alertdef, family=f, action=CREATE)
+2015-11-06 13:14:00,196 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-06 13:14:00,196 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-06 13:14:00,212 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access denied for user root; reason: Global check failed; remote address: /192.168.56.101; request: createTable; context: (user=root, scope=ipzone, family=f, action=CREATE)
+2015-11-06 13:14:00,212 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access denied for user root; reason: Global check failed; remote address: /192.168.56.101; request: createTable; context: (user=root, scope=ipzone, family=f, action=CREATE)
+2015-11-06 13:14:00,240 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-07 08:02:56,907 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Global check allowed; remote address: /192.168.56.101; request: createTable; context: (user=root, scope=alertdef, family=f, action=CREATE)
+2015-11-07 08:02:56,907 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Global check allowed; remote address: /192.168.56.101; request: createTable; context: (user=root, scope=alertdef, family=f, action=CREATE)
+2015-11-07 08:02:56,939 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user hbase; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=hbase, scope=hbase:meta, family=info, action=READ)
+2015-11-07 08:02:56,939 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user hbase; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=hbase, scope=hbase:meta, family=info, action=READ)
+2015-11-07 08:02:56,974 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-07 08:02:56,974 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-07 08:02:57,090 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)
+2015-11-07 12:37:32,439 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Table permission granted; remote address: /192.168.56.101; request: put; context: (user=root, scope=alertStreamSchema, family=f:e|f:b|f:c|f:a|f:attrName|f:dataSource|f:streamName, action=WRITE)
+2015-11-07 12:37:32,439 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Table permission granted; remote address: /192.168.56.101; request: put; context: (user=root, scope=alertStreamSchema, family=f:e|f:b|f:c|f:a|f:attrName|f:dataSource|f:streamName, action=WRITE)
+2015-11-07 12:37:32,439 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Table permission granted; remote address: /192.168.56.101; request: put; context: (user=root, scope=alertStreamSchema, family=f:e|f:b|f:c|f:a|f:attrName|f:dataSource|f:streamName, action=WRITE)
+2015-11-07 12:37:32,439 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Table permission granted; remote address: /192.168.56.101; request: put; context: (user=root, scope=alertStreamSchema, family=f:e|f:b|f:c|f:a|f:attrName|f:dataSource|f:streamName, action=WRITE)
+2015-11-07 12:37:33,639 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Table permission granted; remote address: /192.168.56.101; request: put; context: (user=root, scope=alertStream, family=f:dataSource|f:site|f:streamName, action=WRITE)
+2015-11-07 12:37:33,639 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: Table permission granted; remote address: /192.168.56.101; request: put; context: (user=root, scope=alertStream, family=f:dataSource|f:site|f:streamName, action=WRITE)
+2015-11-07 12:38:05,649 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user hbase; reason: Global check allowed; remote address: ; request: balance; context: (user=hbase, scope=GLOBAL, family=, action=ADMIN)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/pom.xml b/eagle-security/eagle-security-hbase-securitylog/pom.xml
deleted file mode 100644
index 2870807..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ /*
-  ~  * Licensed to the Apache Software Foundation (ASF) under one or more
-  ~  * contributor license agreements.  See the NOTICE file distributed with
-  ~  * this work for additional information regarding copyright ownership.
-  ~  * The ASF licenses this file to You under the Apache License, Version 2.0
-  ~  * (the "License"); you may not use this file except in compliance with
-  ~  * the License.  You may obtain a copy of the License at
-  ~  *
-  ~  *    http://www.apache.org/licenses/LICENSE-2.0
-  ~  *
-  ~  * Unless required by applicable law or agreed to in writing, software
-  ~  * distributed under the License is distributed on an "AS IS" BASIS,
-  ~  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~  * See the License for the specific language governing permissions and
-  ~  * limitations under the License.
-  ~  */
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>eagle-security-parent</artifactId>
-        <groupId>org.apache.eagle</groupId>
-        <version>0.5.0-incubating-SNAPSHOT</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>eagle-security-hbase-securitylog</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-security-common</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-stream-application-manager</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-    </dependencies>
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringTopology.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringTopology.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringTopology.java
deleted file mode 100644
index 0077912..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogMonitoringTopology.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-
-package org.apache.eagle.security.hbase;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.datastream.ExecutionEnvironments;
-import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
-import org.apache.eagle.security.hbase.sensitivity.HbaseResourceSensitivityDataJoinExecutor;
-import org.apache.eagle.stream.application.TopologyExecutable;
-
-
-public class HbaseAuditLogMonitoringTopology implements TopologyExecutable {
-    @Override
-    public void submit(String application, Config config) {
-        //Config baseConfig = ConfigFactory.load();
-        //config = (config != null) ? config.withFallback(baseConfig): baseConfig;
-        StormExecutionEnvironment env = ExecutionEnvironments.getStorm(config);
-        env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1).nameAs("kafkaMsgConsumer")
-                .flatMap(new HbaseResourceSensitivityDataJoinExecutor())
-                .alertWithConsumer("hbaseSecurityLogEventStream", "hbaseSecurityLogAlertExecutor");
-        env.execute();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
deleted file mode 100644
index b00f39c..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase;
-
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.datastream.ExecutionEnvironments;
-import org.apache.eagle.datastream.storm.StormExecutionEnvironment;
-import org.apache.eagle.security.hbase.sensitivity.HbaseResourceSensitivityDataJoinExecutor;
-
-public class HbaseAuditLogProcessorMain {
-    public static void main(String[] args) throws Exception{
-        StormExecutionEnvironment env = ExecutionEnvironments.getStorm(args);
-        env.fromSpout(new KafkaSourcedSpoutProvider()).withOutputFields(1).nameAs("kafkaMsgConsumer")
-                .flatMap(new HbaseResourceSensitivityDataJoinExecutor())
-                .alertWithConsumer("hbaseSecurityLogEventStream", "hbaseSecurityLogAlertExecutor");
-        env.execute();
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogKafkaDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogKafkaDeserializer.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogKafkaDeserializer.java
deleted file mode 100644
index ff98ed9..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogKafkaDeserializer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase.parse;
-
-
-import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-
-
-public class HbaseAuditLogKafkaDeserializer implements SpoutKafkaMessageDeserializer {
-    private static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogKafkaDeserializer.class);
-    private Properties props;
-
-    public HbaseAuditLogKafkaDeserializer(Properties props){
-        this.props = props;
-    }
-
-    @Override
-    public Object deserialize(byte[] arg0) {
-        String logLine = new String(arg0);
-
-        HbaseAuditLogParser parser = new HbaseAuditLogParser();
-        try{
-            HbaseAuditLogObject entity = parser.parse(logLine);
-            if(entity == null) return null;
-
-            Map<String, Object> map = new TreeMap<String, Object>();
-            map.put("action", entity.action);
-            map.put("host", entity.host);
-            map.put("status", entity.status);
-            map.put("request", entity.request);
-            map.put("scope", entity.scope);
-            map.put("user", entity.user);
-            map.put("timestamp", entity.timestamp);
-            return map;
-        }catch(Exception ex){
-            LOG.error("Failing parse audit log:" + logLine, ex);
-            return null;
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogObject.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogObject.java
deleted file mode 100644
index 294f024..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogObject.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.hbase.parse;
-
-
-public class HbaseAuditLogObject {
-    public long timestamp;
-    public String user = "";
-    public String scope = "";
-    public String action = "";
-    public String host = "";
-    public String request = "";
-    public String status = "";
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java
deleted file mode 100644
index 0504ed0..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/parse/HbaseAuditLogParser.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase.parse;
-
-import java.io.Serializable;
-import java.text.ParseException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.eagle.common.DateTimeUtil;
-import org.apache.eagle.security.util.LogParseUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class HbaseAuditLogParser implements Serializable {
-    private final static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogParser.class);
-
-    private final static int LOGDATE_INDEX = 1;
-    private final static int LOGLEVEL_INDEX = 2;
-    private final static int LOGATTRS_INDEX = 3;
-    private final static String ALLOWED = "allowed";
-    private final static String DENIED = "denied";
-    private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*)");
-    private final static Pattern loggerContextPattern = Pattern.compile("\\w+:\\s*\\(user=(.*),\\s*scope=(.*),\\s*family=(.*),\\s*action=(.*)\\)");
-    private final static Pattern allowedPattern = Pattern.compile(ALLOWED);
-
-
-    public HbaseAuditLogObject parse(String logLine) {
-        if(logLine == null || logLine.isEmpty()) return null;
-
-        HbaseAuditLogObject ret = new HbaseAuditLogObject();
-        String timestamp = "";
-        String user = "";
-        String scope = "";
-        String action = "";
-        String ip = "";
-        String request = "";
-        String family = "";
-        String context = "";
-
-        Matcher loggerMatcher = loggerPattern.matcher(logLine);
-        if(loggerMatcher.find()) {
-            try {
-                timestamp = loggerMatcher.group(LOGDATE_INDEX);
-                String [] attrs = loggerMatcher.group(LOGATTRS_INDEX).split(";");
-                ret.status = allowedPattern.matcher(attrs[0]).find() ? ALLOWED : DENIED;
-                try {
-                    ip = attrs[2].split(":")[1].trim();
-                } catch (Exception e) {
-                    ip = "";
-                }
-                try {
-                    request = attrs[3].split(":")[1].trim();
-                } catch (Exception e) {
-                    request = "";
-                }
-                try {
-                    context = attrs[4].trim();
-                } catch (Exception e) {
-                    context = "";
-                }
-                Matcher contextMatcher = loggerContextPattern.matcher(context);
-                if(contextMatcher.find()) {
-                    user = contextMatcher.group(1);
-                    scope = contextMatcher.group(2);
-                    family = contextMatcher.group(3);
-                    action = contextMatcher.group(4);
-                }
-                if(StringUtils.isNotEmpty(family)) {
-                    if(!scope.contains(":")) scope = "default:" + scope;
-                    scope = String.format("%s:%s", scope, family);
-                }
-                if(StringUtils.isNotEmpty(ip)) {
-                    ret.host = ip.substring(1);
-                }
-                ret.timestamp = DateTimeUtil.humanDateToMilliseconds(timestamp);
-                ret.scope = scope;
-                ret.action = action;
-                ret.user = LogParseUtil.parseUserFromUGI(user);
-                ret.request = request;
-                return ret;
-            } catch(Exception e) {
-                LOG.error("Got exception when parsing audit log:" + logLine + ", exception:" + e.getMessage(), e);
-            }
-        }
-        return null;
-    }
-}
-
-
-

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java
deleted file mode 100644
index 82db688..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityDataJoinExecutor.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase.sensitivity;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.datastream.JavaStormStreamExecutor2;
-import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.apache.eagle.security.util.ExternalDataJoiner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.regex.Pattern;
-
-public class HbaseResourceSensitivityDataJoinExecutor extends JavaStormStreamExecutor2<String, Map> {
-    private final static Logger LOG = LoggerFactory.getLogger(
-            HbaseResourceSensitivityDataJoinExecutor.class);
-    private Config config;
-
-    @Override
-    public void prepareConfig(Config config) {
-        this.config = config;
-    }
-
-    @Override
-    public void init() {
-        // start hive resource data polling
-        try {
-            ExternalDataJoiner joiner = new ExternalDataJoiner(
-                    HbaseResourceSensitivityPollingJob.class, config);
-            joiner.start();
-        } catch(Exception ex){
-            LOG.error("Fail to bring up quartz scheduler.", ex);
-            throw new IllegalStateException(ex);
-        }
-    }
-
-    @Override
-    public void flatMap(List<Object> input, Collector<Tuple2<String, Map>> outputCollector){
-        @SuppressWarnings("unchecked")
-        Map<String, Object> event = (Map<String, Object>)input.get(0);
-        @SuppressWarnings("unchecked")
-        Map<String, HbaseResourceSensitivityAPIEntity> map =
-                (Map<String, HbaseResourceSensitivityAPIEntity>) ExternalDataCache
-                        .getInstance()
-                        .getJobResult(HbaseResourceSensitivityPollingJob.class);
-        LOG.info(">>>> event: " + event + " >>>> map: " + map);
-
-        String resource = (String)event.get("scope");
-
-        HbaseResourceSensitivityAPIEntity sensitivityEntity = null;
-
-        if (map != null && resource != "") {
-            for (String key : map.keySet()) {
-                Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
-                if(pattern.matcher(resource).find()) {
-                    sensitivityEntity = map.get(key);
-                    break;
-                }
-            }
-        }
-        Map<String, Object> newEvent = new TreeMap<String, Object>(event);
-        newEvent.put("sensitivityType", sensitivityEntity  == null ?
-                "NA" : sensitivityEntity.getSensitivityType());
-        newEvent.put("scope", resource);
-        if(LOG.isDebugEnabled()) {
-            LOG.debug("After hbase resource sensitivity lookup: " + newEvent);
-        }
-        LOG.info("After hbase resource sensitivity lookup: " + newEvent);
-        outputCollector.collect(new Tuple2(
-                newEvent.get("user"),
-                newEvent));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityPollingJob.java b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityPollingJob.java
deleted file mode 100644
index a5d3978..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/sensitivity/HbaseResourceSensitivityPollingJob.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase.sensitivity;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Maps;
-import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
-import org.apache.eagle.security.util.AbstractResourceSensitivityPollingJob;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.quartz.Job;
-import org.quartz.JobDataMap;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-public class HbaseResourceSensitivityPollingJob extends AbstractResourceSensitivityPollingJob implements Job {
-    private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityPollingJob.class);
-
-    @Override
-    public void execute(JobExecutionContext context)
-            throws JobExecutionException {
-        JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
-        try {
-            List<HbaseResourceSensitivityAPIEntity>
-            hbaseResourceSensitivity = load(jobDataMap, "HbaseResourceSensitivityService");
-            if(hbaseResourceSensitivity == null) {
-            	LOG.warn("Hbase resource sensitivity information is empty");
-            	return;
-            }
-            Map<String, HbaseResourceSensitivityAPIEntity> map = Maps.uniqueIndex(
-            		hbaseResourceSensitivity,
-            		new Function<HbaseResourceSensitivityAPIEntity, String>() {
-            			@Override
-            			public String apply(HbaseResourceSensitivityAPIEntity input) {
-            				return input.getTags().get("hbaseResource");
-            			}
-            		});
-            ExternalDataCache.getInstance().setJobResult(getClass(), map);
-        } catch(Exception ex) {
-        	LOG.error("Fail to load hbase resource sensitivity data", ex);
-        }
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/resources/application.conf b/eagle-security/eagle-security-hbase-securitylog/src/main/resources/application.conf
deleted file mode 100644
index e92cb4f..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/resources/application.conf
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
-  "envContextConfig" : {
-    "env" : "storm",
-    "mode" : "local",
-    "topologyName" : "sandbox-hbaseSecurityLog-topology",
-    "stormConfigFile" : "security-auditlog-storm.yaml",
-    "parallelismConfig" : {
-      "kafkaMsgConsumer" : 1,
-      "hbaseSecurityLogAlertExecutor*" : 1
-    }
-  },
-  "dataSourceConfig": {
-    "topic" : "sandbox_hbase_security_log",
-    "zkConnection" : "sandbox.hortonworks.com:2181",
-    "zkConnectionTimeoutMS" : 15000,
-    "consumerGroupId" : "EagleConsumer",
-    "fetchSize" : 1048586,
-    "deserializerClass" : "org.apache.eagle.security.hbase.parse.HbaseAuditLogKafkaDeserializer",
-    "transactionZKServers" : "sandbox.hortonworks.com",
-    "transactionZKPort" : 2181,
-    "transactionZKRoot" : "/consumers",
-    "consumerGroupId" : "eagle.hbasesecurity.consumer",
-    "transactionStateUpdateMS" : 2000
-  },
-  "alertExecutorConfigs" : {
-     "hbaseSecurityLogAlertExecutor" : {
-       "parallelism" : 1,
-       "partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner"
-       "needValidation" : "true"
-     }
-  },
-  "eagleProps" : {
-    "site" : "sandbox",
-    "application": "hbaseSecurityLog",
-    "dataJoinPollIntervalSec" : 30,
-    "mailHost" : "mailHost.com",
-    "mailSmtpPort":"25",
-    "mailDebug" : "true",
-    "eagleService": {
-      "host": "localhost",
-      "port": 9098
-      "username": "admin",
-      "password": "secret"
-    }
-  },
-  "dynamicConfigSource" : {
-    "enabled" : true,
-    "initDelayMillis" : 0,
-    "delayMillis" : 30000
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/resources/log4j.properties b/eagle-security/eagle-security-hbase-securitylog/src/main/resources/log4j.properties
deleted file mode 100644
index fb13ad5..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=DEBUG, stdout
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
deleted file mode 100644
index 326450d..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase;
-
-import org.apache.eagle.security.hbase.parse.HbaseAuditLogObject;
-import org.apache.eagle.security.hbase.parse.HbaseAuditLogParser;
-import org.junit.Assert;
-import org.junit.Test;
-
-
-public class TestHbaseAuditLogParser {
-    HbaseAuditLogParser parser = new HbaseAuditLogParser();
-
-    @Test
-    public void test() throws Exception {
-        String log = "2015-08-11 13:31:03,729 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Table permission granted; remote address: /127.0.0.1; request: get; context: (user=eagle,scope=hbase:namespace,family=info, action=READ)";
-        HbaseAuditLogObject obj = parser.parse(log);
-        Assert.assertEquals(obj.action, "READ");
-        Assert.assertEquals(obj.host, "127.0.0.1");
-        Assert.assertEquals(obj.scope, "hbase:namespace:info");
-    }
-
-    @Test
-    public void test2() throws Exception {
-        String log = "2015-08-04 12:29:03,073 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Global check allowed; remote address: ; request: preOpen; context: (user=eagle, scope=GLOBAL, family=, action=ADMIN)";
-        HbaseAuditLogObject obj = parser.parse(log);
-        Assert.assertEquals(obj.action, "ADMIN");
-        Assert.assertEquals(obj.host, "");
-        Assert.assertEquals(obj.scope, "GLOBAL");
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/eef4930c/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java b/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java
deleted file mode 100644
index c1be351..0000000
--- a/eagle-security/eagle-security-hbase-securitylog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogProcessTopology.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-
-package org.apache.eagle.security.hbase;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import com.typesafe.config.ConfigSyntax;
-import org.apache.eagle.common.config.EagleConfigConstants;
-import org.junit.Test;
-
-
-public class TestHbaseAuditLogProcessTopology {
-    @Test
-    public void test() throws Exception {
-        //Config baseConfig = ConfigFactory.load("eagle-scheduler.conf");
-        ConfigParseOptions options = ConfigParseOptions.defaults()
-                .setSyntax(ConfigSyntax.PROPERTIES)
-                .setAllowMissing(false);
-        String topoConfigStr = "web.hbase.zookeeper.property.clientPort=2181\nweb.hbase.zookeeper.quorum=sandbox.hortonworks.com\n\napp.envContextConfig.env=storm\napp.envContextConfig.mode=local\napp.dataSourceConfig.topic=sandbox_hbase_security_log\napp.dataSourceConfig.zkConnection=sandbox.hortonworks.com:2181\napp.dataSourceConfig.zkConnectionTimeoutMS=15000\napp.dataSourceConfig.brokerZkPath=/brokers\napp.dataSourceConfig.fetchSize=1048586\napp.dataSourceConfig.transactionZKServers=sandbox.hortonworks.com\napp.dataSourceConfig.transactionZKPort=2181\napp.dataSourceConfig.transactionZKRoot=/consumers\napp.dataSourceConfig.consumerGroupId=eagle.hbasesecurity.consumer\napp.dataSourceConfig.transactionStateUpdateMS=2000\napp.dataSourceConfig.deserializerClass=org.apache.eagle.security.hbase.parse.HbaseAuditLogKafkaDeserializer\napp.eagleProps.site=sandbox\napp.eagleProps.application=hbaseSecurityLog\napp.eagleProps.dataJoinPollIntervalSec=30\napp.eagleProps.mailHost=mailHost.com\na
 pp.eagleProps.mailSmtpPort=25\napp.eagleProps.mailDebug=true\napp.eagleProps.eagleService.host=localhost\napp.eagleProps.eagleService.port=9098\napp.eagleProps.eagleService.username=admin\napp.eagleProps.eagleService.password=secret";
-
-        Config topoConfig = ConfigFactory.parseString(topoConfigStr, options);
-        Config conf = topoConfig.getConfig(EagleConfigConstants.APP_CONFIG);
-
-        HbaseAuditLogMonitoringTopology topology = new HbaseAuditLogMonitoringTopology();
-        //topology.submit("", conf);
-    }
-}



Mime
View raw message