eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject [2/2] incubator-eagle git commit: EAGLE-470 data enrichment refactor in eagle security monitoring data enrichment refactor in eagle security monitoring https://issues.apache.org/jira/browse/EAGLE-470
Date Wed, 17 Aug 2016 17:07:49 GMT
EAGLE-470 data enrichment refactor in eagle security monitoring
data enrichment refactor in eagle security monitoring
https://issues.apache.org/jira/browse/EAGLE-470

Author: @yonzhang <yonzhang2012@gmail.com>

Closes: #358


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/92ca7c54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/92ca7c54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/92ca7c54

Branch: refs/heads/develop
Commit: 92ca7c54233a71689da8cffd01e282ee8f8de05e
Parents: 48bb082
Author: yonzhang <yonzhang2012@gmail.com>
Authored: Wed Aug 17 10:02:41 2016 -0700
Committer: yonzhang <yonzhang2012@gmail.com>
Committed: Wed Aug 17 10:02:41 2016 -0700

----------------------------------------------------------------------
 .../src/main/resources/log4j.properties         |   4 +-
 .../security/enrich/AbstractDataEnrichBolt.java |  67 ++++++++
 .../security/enrich/AbstractDataEnrichLCM.java  |  51 ++++++
 .../eagle/security/enrich/DataEnrichJob.java    |  55 +++++++
 .../eagle/security/enrich/DataEnrichLCM.java    |  45 ++++++
 .../security/enrich/ExternalDataCache.java      |  41 +++++
 .../security/enrich/ExternalDataJoiner.java     |  99 ++++++++++++
 .../entity/FileSensitivityAPIEntity.java        |  52 ------
 .../HbaseResourceSensitivityAPIEntity.java      |  47 ------
 .../HiveResourceSensitivityAPIEntity.java       |  53 ------
 .../eagle/security/entity/IPZoneEntity.java     |  52 ------
 .../entity/SecurityEntityRepository.java        |   5 -
 .../service/IMetadataServiceClient.java         |  38 -----
 .../ISecurityDataEnrichServiceClient.java       |  38 +++++
 .../service/MetadataServiceClientImpl.java      | 160 -------------------
 .../SecurityDataEnrichServiceClientImpl.java    | 160 +++++++++++++++++++
 .../AbstractResourceSensitivityPollingJob.java  |  56 -------
 .../eagle/security/util/ExternalDataCache.java  |  43 -----
 .../eagle/security/util/ExternalDataJoiner.java | 109 -------------
 .../hbase/HBaseSensitivityDataEnrichLCM.java    |  43 +++++
 .../HbaseResourceSensitivityDataJoinBolt.java   |  41 +----
 .../HbaseResourceSensitivityPollingJob.java     |  72 ---------
 .../AbstractHdfsAuditLogApplication.java        |   4 +-
 .../auditlog/FileSensitivityDataJoinBolt.java   | 108 -------------
 .../auditlog/HdfsSensitivityDataEnrichBolt.java |  82 ++++++++++
 .../auditlog/HdfsSensitivityDataEnrichLCM.java  |  43 +++++
 .../security/auditlog/IPZoneDataEnrichBolt.java |  62 +++++++
 .../security/auditlog/IPZoneDataEnrichLCM.java  |  43 +++++
 .../security/auditlog/IPZoneDataJoinBolt.java   |  83 ----------
 .../timer/FileSensitivityPollingJob.java        |  74 ---------
 .../auditlog/timer/IPZonePollingJob.java        |  77 ---------
 .../hdfs/HDFSResourceSensitivityDataJoiner.java |   8 +-
 .../hdfs/HDFSResourceSensitivityService.java    | 143 ++++++++---------
 .../hdfs/rest/HDFSResourceWebResource.java      |   3 +-
 .../hdfs/test/HDFSChildSensitivityTypeTest.java |   8 +-
 .../hive/HiveQueryMonitoringApplication.java    |   4 +-
 .../HiveResourceSensitivityDataJoinBolt.java    | 105 ------------
 .../HiveResourceSensitivityPollingJob.java      |  83 ----------
 .../HiveSensitivityDataEnrichBolt.java          |  82 ++++++++++
 .../HiveSensitivityDataEnrichLCM.java           |  43 +++++
 .../hdfs/rest/MAPRFSResourceWebResource.java    |   4 +-
 .../OozieResourceSensitivityDataJoinBolt.java   |  34 +---
 .../OozieResourceSensitivityPollingJob.java     |  63 --------
 .../OozieSensitivityDataEnrichLCM.java          |  42 +++++
 44 files changed, 1094 insertions(+), 1435 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties b/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
index 6c31ebc..f6076a6 100644
--- a/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
+++ b/eagle-security/eagle-metric-collection/src/main/resources/log4j.properties
@@ -19,8 +19,8 @@ eagle.log.dir=./logs
 eagle.log.file=eagle.log
 
 
-#log4j.logger.eagle.security.auditlog.IPZoneDataJoinBolt=DEBUG
-#log4j.logger.eagle.security.auditlog.FileSensitivityDataJoinBolt=DEBUG
+#log4j.logger.eagle.security.auditlog.IPZoneDataEnrichBolt=DEBUG
+#log4j.logger.eagle.security.auditlog.HdfsSensitivityDataEnrichBolt=DEBUG
 #log4j.logger.eagle.executor.AlertExecutor=DEBUG
 # standard output
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java
new file mode 100644
index 0000000..6c23180
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichBolt.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Since 8/16/16.
+ */
+public abstract class AbstractDataEnrichBolt<T, K> extends BaseRichBolt {
+    private final static Logger LOG = LoggerFactory.getLogger(AbstractDataEnrichBolt.class);
+    protected Config config;
+    protected OutputCollector collector;
+    private DataEnrichLCM<T, K> lcm;
+
+    public AbstractDataEnrichBolt(Config config, DataEnrichLCM<T, K> lcm){
+        this.config = config;
+        this.lcm = lcm;
+    }
+
+    @Override
+    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+        this.collector = collector;
+        // start external data retrieval
+        try {
+            ExternalDataJoiner joiner = new ExternalDataJoiner(
+                    lcm, config, context.getThisComponentId() + "." + context.getThisTaskIndex());
+            joiner.start();
+        } catch(Exception ex){
+            LOG.error("Fail bringing up quartz scheduler.", ex);
+            throw new IllegalStateException(ex);
+        }
+    }
+
+    protected abstract void executeWithEnrich(Tuple input, Map<K, T> cache);
+
+    @Override
+    public void execute(Tuple input) {
+        Map map = (Map) ExternalDataCache
+                    .getInstance()
+                    .getJobResult(lcm.getClass());
+        executeWithEnrich(input, map);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java
new file mode 100644
index 0000000..002bf82
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/AbstractDataEnrichLCM.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.service.ISecurityDataEnrichServiceClient;
+import org.apache.eagle.security.service.SecurityDataEnrichServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * Since 8/16/16.
+ */
+public abstract class AbstractDataEnrichLCM<T, K> implements DataEnrichLCM<T, K> {
+    private static final Logger LOG = LoggerFactory.getLogger(AbstractDataEnrichLCM.class);
+    private Config config;
+    public AbstractDataEnrichLCM(Config config){
+        this.config = config;
+    }
+
+    @Override
+    public Collection<T> loadExternal() {
+        String eagleServiceHost = config.getString("eagleService.host");
+        Integer eagleServicePort = config.getInt("eagleService.port");
+
+        // load from eagle database
+        LOG.info("Load sensitivity information from eagle service " + eagleServiceHost + ":" + eagleServicePort);
+
+        ISecurityDataEnrichServiceClient client = new SecurityDataEnrichServiceClientImpl(eagleServiceHost, eagleServicePort, "/rest");
+        return loadFromService(client);
+    }
+
+    protected abstract Collection<T> loadFromService(ISecurityDataEnrichServiceClient client);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java
new file mode 100644
index 0000000..13777bf
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichJob.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import com.google.common.collect.Maps;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Since 8/16/16.
+ */
+public class DataEnrichJob implements Job {
+    private final static Logger LOG = LoggerFactory.getLogger(DataEnrichJob.class);
+
+    @Override
+    public void execute(JobExecutionContext context) throws JobExecutionException {
+        JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
+
+        DataEnrichLCM lcm = (DataEnrichLCM)jobDataMap.getOrDefault("dataEnrichLCM", null);
+        if(lcm == null)
+            throw new IllegalStateException("dataEnrichLCM implementation should be provided");
+        try {
+            Collection externalEntities = lcm.loadExternal();
+            Map<Object, Object> map = Maps.uniqueIndex(
+                    externalEntities,
+                    entity -> lcm.getCacheKey(entity)
+                );
+            ExternalDataCache.getInstance().setJobResult(lcm.getClass(), map);
+        } catch(Exception ex) {
+            LOG.error("Fail to load hbase resource sensitivity data", ex);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java
new file mode 100644
index 0000000..58fecf1
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/DataEnrichLCM.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Since 8/16/16.
+ * Data enrichment lifecycle methods
+ * @param <T> entity type, this entity will be put into cache
+ * @param <K> cache key
+ */
+public interface DataEnrichLCM<T, K> extends Serializable{
+    /**
+     * load all external data for real time enrichment
+     *
+     * @return
+     */
+    Collection<T> loadExternal();
+
+    /**
+     * get cache key from one entity
+     *
+     * @param entity
+     * @return
+     */
+    K getCacheKey(T entity);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java
new file mode 100644
index 0000000..0293b30
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataCache.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.enrich;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ExternalDataCache {
+	private Map<Class<? extends DataEnrichLCM>, Object> map = new ConcurrentHashMap<>();
+	
+	private static final ExternalDataCache instance = new ExternalDataCache();
+	
+	private ExternalDataCache(){
+	}
+	
+	public static ExternalDataCache getInstance(){
+		return instance;
+	}
+	
+	public Object getJobResult(Class<? extends DataEnrichLCM> cls){
+		return map.get(cls);
+	}
+	
+	public void setJobResult(Class<? extends DataEnrichLCM> cls, Object jobResult){
+		map.put(cls, jobResult);
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java
new file mode 100644
index 0000000..1f593a8
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/enrich/ExternalDataJoiner.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.enrich;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.util.JVMSingleQuartzScheduler;
+import org.quartz.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * spawn Quartz scheduler to poll data from eagle service and invoke custom job to handle
+ */
+public class ExternalDataJoiner {
+	private static final Logger LOG = LoggerFactory.getLogger(ExternalDataJoiner.class);
+
+	private Scheduler sched;
+	private JobDataMap jobDataMap;
+	private String jobId;
+	private String taskId;
+	private Config config;
+
+	private static final String DATA_JOIN_POLL_INTERVALSEC = "dataJoinPollIntervalSec";
+	private static final String QUARTZ_GROUP_NAME = "dataJoiner";
+
+	private final int defaultIntervalSeconds = 60;
+
+	/**
+	 * constructor with parameters which are passed from customer
+	 * throws Exception while this scheduler can't be brought up
+	 * Some protocols:
+	 * 1. prop should include an entry keyed by "data.join.poll.intervalInSeconds" if you want to customize poll interval.
+	 */
+	public ExternalDataJoiner(DataEnrichLCM lcm, Config config, String taskId) throws Exception{
+		this.taskId = taskId;
+		Map<String, Object> map = new HashMap<>();
+		// add DataEnrichLCM to JobDataMap
+		map.put("dataEnrichLCM", lcm);
+		this.config = config;
+		init(lcm.getClass().getCanonicalName(), map);
+	}
+
+	private void init(String jobId, Map<String, Object> prop) throws Exception{
+		this.jobId = jobId;
+		jobDataMap = new JobDataMap(prop);
+		sched = JVMSingleQuartzScheduler.getInstance().getScheduler();
+	}
+
+	public void start(){
+		// for job
+		String group = String.format("%s.%s.%s.%s",
+				QUARTZ_GROUP_NAME,
+				config.hasPath("siteId") ? config.getString("siteId") : "",
+				config.hasPath("appId") ? config.getString("appId") : "",
+				taskId);
+		JobDetail job = JobBuilder.newJob(DataEnrichJob.class)
+		     .withIdentity(jobId + ".job", group)
+		     .usingJobData(jobDataMap)
+		     .build();
+
+		// for trigger
+		Object interval = jobDataMap.get(DATA_JOIN_POLL_INTERVALSEC);
+        int dataJoinPollIntervalSec = (interval == null ? defaultIntervalSeconds : Integer.parseInt(interval.toString()));
+		Trigger trigger = TriggerBuilder.newTrigger()
+			  .withIdentity(jobId + ".trigger", group)
+		      .startNow()
+		      .withSchedule(SimpleScheduleBuilder.simpleSchedule()
+		          .withIntervalInSeconds(dataJoinPollIntervalSec)
+		          .repeatForever())
+		      .build();
+		try{
+			sched.scheduleJob(job, trigger);
+		}catch(Exception ex){
+			LOG.error("Can't schedule job " + job.getDescription(), ex);
+		}
+	}
+
+	public void stop() throws Exception{
+		sched.shutdown();
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java
deleted file mode 100644
index 9eef192..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/FileSensitivityAPIEntity.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.entity;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.Column;
-import org.apache.eagle.log.entity.meta.ColumnFamily;
-import org.apache.eagle.log.entity.meta.Prefix;
-import org.apache.eagle.log.entity.meta.Service;
-import org.apache.eagle.log.entity.meta.Table;
-import org.apache.eagle.log.entity.meta.Tags;
-import org.apache.eagle.log.entity.meta.TimeSeries;
-
-@JsonSerialize(include =  JsonSerialize.Inclusion.NON_NULL)
-@Table("fileSensitivity")
-@ColumnFamily("f")
-@Prefix("fileSensitivity")
-@Service("FileSensitivityService")
-@TimeSeries(false)
-@Tags({"site", "filedir"})
-public class FileSensitivityAPIEntity extends TaggedLogAPIEntity{
-	/**
-	 * sensitivityType can be multi-value attribute, and values can be separated by "|"
-	 */
-	@Column("a")
-	private String sensitivityType;
-
-	public String getSensitivityType() {
-		return sensitivityType;
-	}
-
-	public void setSensitivityType(String sensitivityType) {
-		this.sensitivityType = sensitivityType;
-		valueChanged("sensitivityType");
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java
deleted file mode 100644
index 551967d..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceSensitivityAPIEntity.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.security.entity;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.*;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Table("hbaseResourceSensitivity")
-@ColumnFamily("f")
-@Prefix("hbaseResourceSensitivity")
-@Service("HbaseResourceSensitivityService")
-@TimeSeries(false)
-@Tags({"site", "hbaseResource"})
-public class HbaseResourceSensitivityAPIEntity extends TaggedLogAPIEntity {
-	private static final long serialVersionUID = 2L;
-	/**
-     * sensitivityType can be multi-value attribute, and values can be separated by "|"
-     */
-    @Column("a")
-    private String sensitivityType;
-
-    public String getSensitivityType() {
-        return sensitivityType;
-    }
-
-    public void setSensitivityType(String sensitivityType) {
-        this.sensitivityType = sensitivityType;
-        valueChanged("sensitivityType");
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java
deleted file mode 100644
index aebec8b..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HiveResourceSensitivityAPIEntity.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.entity;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.Column;
-import org.apache.eagle.log.entity.meta.ColumnFamily;
-import org.apache.eagle.log.entity.meta.Prefix;
-import org.apache.eagle.log.entity.meta.Service;
-import org.apache.eagle.log.entity.meta.Table;
-import org.apache.eagle.log.entity.meta.Tags;
-import org.apache.eagle.log.entity.meta.TimeSeries;
-
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Table("hiveResourceSensitivity")
-@ColumnFamily("f")
-@Prefix("hiveResourceSensitivity")
-@Service("HiveResourceSensitivityService")
-@TimeSeries(false)
-@Tags({"site", "hiveResource"})
-public class HiveResourceSensitivityAPIEntity extends TaggedLogAPIEntity {
-	private static final long serialVersionUID = 1L;
-	/**
-     * sensitivityType can be multi-value attribute, and values can be separated by "|"
-     */
-    @Column("a")
-    private String sensitivityType;
-
-    public String getSensitivityType() {
-        return sensitivityType;
-    }
-
-    public void setSensitivityType(String sensitivityType) {
-        this.sensitivityType = sensitivityType;
-        valueChanged("sensitivityType");
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java
deleted file mode 100644
index f944ace..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/IPZoneEntity.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.entity;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.meta.Column;
-import org.apache.eagle.log.entity.meta.ColumnFamily;
-import org.apache.eagle.log.entity.meta.Prefix;
-import org.apache.eagle.log.entity.meta.Service;
-import org.apache.eagle.log.entity.meta.Table;
-import org.apache.eagle.log.entity.meta.Tags;
-import org.apache.eagle.log.entity.meta.TimeSeries;
-
-/**
- * ipranger can be single IP, host or ip with subnet etc.  
- */
-@JsonSerialize(include =  JsonSerialize.Inclusion.NON_NULL)
-@Table("ipzone")
-@ColumnFamily("f")
-@Prefix("ipzone")
-@Service("IPZoneService")
-@TimeSeries(false)
-@Tags({"iphost"})
-public class IPZoneEntity extends TaggedLogAPIEntity{
-	@Column("a")
-	private String securityZone;
-
-	public String getSecurityZone() {
-		return securityZone;
-	}
-
-	public void setSecurityZone(String securityZone) {
-		this.securityZone = securityZone;
-		valueChanged("securityZone");
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
index 5d63690..7c51724 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/SecurityEntityRepository.java
@@ -23,11 +23,6 @@ import org.apache.eagle.log.entity.repo.EntityRepository;
 
 public class SecurityEntityRepository extends EntityRepository {
     public SecurityEntityRepository() {
-        entitySet.add(HbaseResourceSensitivityAPIEntity.class);
-        entitySet.add(FileSensitivityAPIEntity.class);
-        entitySet.add(IPZoneEntity.class);
         entitySet.add(HdfsUserCommandPatternEntity.class);
-        entitySet.add(HiveResourceSensitivityAPIEntity.class);
-        entitySet.add(OozieResourceSensitivityAPIEntity.class);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java
deleted file mode 100644
index d87cf13..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/IMetadataServiceClient.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- *
- *  * Licensed to the Apache Software Foundation (ASF) under one or more
- *  * contributor license agreements.  See the NOTICE file distributed with
- *  * this work for additional information regarding copyright ownership.
- *  * The ASF licenses this file to You under the Apache License, Version 2.0
- *  * (the "License"); you may not use this file except in compliance with
- *  * the License.  You may obtain a copy of the License at
- *  * <p/>
- *  * http://www.apache.org/licenses/LICENSE-2.0
- *  * <p/>
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
- *
- */
-
-package org.apache.eagle.security.service;
-
-import java.io.Closeable;
-import java.io.Serializable;
-import java.util.Collection;
-
-/**
- * service stub to get metadata from remote metadata service
- */
-public interface IMetadataServiceClient extends Closeable, Serializable {
-    Collection<HBaseSensitivityEntity> listHBaseSensitivities();
-    OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h);
-    Collection<HdfsSensitivityEntity> listHdfsSensitivities();
-    OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h);
-    Collection<IPZoneEntity> listIPZones();
-    OpResult addIPZone(Collection<IPZoneEntity> h);
-    Collection<HiveSensitivityEntity> listHiveSensitivities();
-    OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h);
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java
new file mode 100644
index 0000000..916b08f
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/ISecurityDataEnrichServiceClient.java
@@ -0,0 +1,38 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  * contributor license agreements.  See the NOTICE file distributed with
+ *  * this work for additional information regarding copyright ownership.
+ *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  * (the "License"); you may not use this file except in compliance with
+ *  * the License.  You may obtain a copy of the License at
+ *  * <p/>
+ *  * http://www.apache.org/licenses/LICENSE-2.0
+ *  * <p/>
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.service;
+
+import java.io.Closeable;
+import java.io.Serializable;
+import java.util.Collection;
+
+/**
+ * service stub to get metadata from remote metadata service
+ */
+public interface ISecurityDataEnrichServiceClient extends Closeable, Serializable {
+    Collection<HBaseSensitivityEntity> listHBaseSensitivities();
+    OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h);
+    Collection<HdfsSensitivityEntity> listHdfsSensitivities();
+    OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h);
+    Collection<IPZoneEntity> listIPZones();
+    OpResult addIPZone(Collection<IPZoneEntity> h);
+    Collection<HiveSensitivityEntity> listHiveSensitivities();
+    OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java
deleted file mode 100644
index c350c2c..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/MetadataServiceClientImpl.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- *
- *  * Licensed to the Apache Software Foundation (ASF) under one or more
- *  * contributor license agreements.  See the NOTICE file distributed with
- *  * this work for additional information regarding copyright ownership.
- *  * The ASF licenses this file to You under the Apache License, Version 2.0
- *  * (the "License"); you may not use this file except in compliance with
- *  * the License.  You may obtain a copy of the License at
- *  * <p/>
- *  * http://www.apache.org/licenses/LICENSE-2.0
- *  * <p/>
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
- *
- */
-
-package org.apache.eagle.security.service;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import com.typesafe.config.Config;
-import org.codehaus.jackson.jaxrs.JacksonJsonProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-public class MetadataServiceClientImpl implements IMetadataServiceClient {
-    private static final long serialVersionUID = 3003976065082684128L;
-
-    private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceClientImpl.class);
-
-    private static final String METADATA_LIST_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
-    private static final String METADATA_ADD_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
-
-    private static final String METADATA_LIST_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
-    private static final String METADATA_ADD_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
-
-    private static final String METADATA_LIST_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
-    private static final String METADATA_ADD_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
-
-    private static final String METADATA_LIST_IPZONE_PATH = "/metadata/security/ipzone";
-    private static final String METADATA_ADD_IPZONE_PATH = "/metadata/security/ipzone";
-
-    private static final String EAGLE_CORRELATION_CONTEXT = "metadataService.context";
-    private static final String EAGLE_CORRELATION_SERVICE_PORT = "metadataService.port";
-    private static final String EAGLE_CORRELATION_SERVICE_HOST = "metadataService.host";
-
-    protected static final String CONTENT_TYPE = "Content-Type";
-
-    private String host;
-    private int port;
-    private String context;
-    private transient Client client;
-    private String basePath;
-
-    public MetadataServiceClientImpl(Config config) {
-        this(config.getString(EAGLE_CORRELATION_SERVICE_HOST), config.getInt(EAGLE_CORRELATION_SERVICE_PORT), config
-                .getString(EAGLE_CORRELATION_CONTEXT));
-        basePath = buildBasePath();
-    }
-
-    public MetadataServiceClientImpl(String host, int port, String context) {
-        this.host = host;
-        this.port = port;
-        this.context = context;
-        this.basePath = buildBasePath();
-        ClientConfig cc = new DefaultClientConfig();
-        cc.getProperties().put(DefaultClientConfig.PROPERTY_CONNECT_TIMEOUT, 60 * 1000);
-        cc.getProperties().put(DefaultClientConfig.PROPERTY_READ_TIMEOUT, 60 * 1000);
-        cc.getClasses().add(JacksonJsonProvider.class);
-        cc.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
-        this.client = Client.create(cc);
-        client.addFilter(new com.sun.jersey.api.client.filter.GZIPContentEncodingFilter());
-    }
-
-    private String buildBasePath() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("http://");
-        sb.append(host);
-        sb.append(":");
-        sb.append(port);
-        sb.append(context);
-        return sb.toString();
-    }
-
-    private <T> List<T> list(String path, GenericType<List<T>> type) {
-        WebResource r = client.resource(basePath + path);
-        LOG.info("query URL {}", basePath + path);
-        List<T> ret = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).get(type);
-        return ret;
-    }
-
-    @Override
-    public void close() throws IOException {
-        client.destroy();
-    }
-
-    @Override
-    public Collection<HBaseSensitivityEntity> listHBaseSensitivities() {
-        return list(METADATA_LIST_HBASE_SENSITIVITY_PATH, new GenericType<List<HBaseSensitivityEntity>>() {
-        });
-    }
-
-    @Override
-    public OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h) {
-        WebResource r = client.resource(basePath + METADATA_ADD_HBASE_SENSITIVITY_PATH);
-        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
-        return new OpResult();
-    }
-
-    @Override
-    public Collection<HdfsSensitivityEntity> listHdfsSensitivities() {
-        return list(METADATA_LIST_HDFS_SENSITIVITY_PATH, new GenericType<List<HdfsSensitivityEntity>>() {
-        });
-    }
-
-    @Override
-    public OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h) {
-        WebResource r = client.resource(basePath + METADATA_ADD_HDFS_SENSITIVITY_PATH);
-        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
-        return new OpResult();
-    }
-
-    @Override
-    public Collection<IPZoneEntity> listIPZones(){
-        return list(METADATA_LIST_IPZONE_PATH, new GenericType<List<IPZoneEntity>>() {
-        });
-    }
-
-    @Override
-    public OpResult addIPZone(Collection<IPZoneEntity> h){
-        WebResource r = client.resource(basePath + METADATA_ADD_IPZONE_PATH);
-        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
-        return new OpResult();
-    }
-
-    @Override
-    public Collection<HiveSensitivityEntity> listHiveSensitivities() {
-        return list(METADATA_LIST_HIVE_SENSITIVITY_PATH, new GenericType<List<HiveSensitivityEntity>>() {
-        });
-    }
-
-    @Override
-    public OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h) {
-        WebResource r = client.resource(basePath + METADATA_ADD_HIVE_SENSITIVITY_PATH);
-        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
-        return new OpResult();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java
new file mode 100644
index 0000000..ffabcbc
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/SecurityDataEnrichServiceClientImpl.java
@@ -0,0 +1,160 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  * contributor license agreements.  See the NOTICE file distributed with
+ *  * this work for additional information regarding copyright ownership.
+ *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  * (the "License"); you may not use this file except in compliance with
+ *  * the License.  You may obtain a copy of the License at
+ *  * <p/>
+ *  * http://www.apache.org/licenses/LICENSE-2.0
+ *  * <p/>
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.service;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import com.typesafe.config.Config;
+import org.codehaus.jackson.jaxrs.JacksonJsonProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+public class SecurityDataEnrichServiceClientImpl implements ISecurityDataEnrichServiceClient {
+    private static final long serialVersionUID = 3003976065082684128L;
+
+    private static final Logger LOG = LoggerFactory.getLogger(SecurityDataEnrichServiceClientImpl.class);
+
+    private static final String METADATA_LIST_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
+    private static final String METADATA_ADD_HBASE_SENSITIVITY_PATH = "/metadata/security/hbaseSensitivity";
+
+    private static final String METADATA_LIST_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
+    private static final String METADATA_ADD_HDFS_SENSITIVITY_PATH = "/metadata/security/hdfsSensitivity";
+
+    private static final String METADATA_LIST_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
+    private static final String METADATA_ADD_HIVE_SENSITIVITY_PATH = "/metadata/security/hiveSensitivity";
+
+    private static final String METADATA_LIST_IPZONE_PATH = "/metadata/security/ipzone";
+    private static final String METADATA_ADD_IPZONE_PATH = "/metadata/security/ipzone";
+
+    private static final String EAGLE_CORRELATION_CONTEXT = "metadataService.context";
+    private static final String EAGLE_CORRELATION_SERVICE_PORT = "metadataService.port";
+    private static final String EAGLE_CORRELATION_SERVICE_HOST = "metadataService.host";
+
+    protected static final String CONTENT_TYPE = "Content-Type";
+
+    private String host;
+    private int port;
+    private String context;
+    private transient Client client;
+    private String basePath;
+
+    public SecurityDataEnrichServiceClientImpl(Config config) {
+        this(config.getString(EAGLE_CORRELATION_SERVICE_HOST), config.getInt(EAGLE_CORRELATION_SERVICE_PORT), config
+                .getString(EAGLE_CORRELATION_CONTEXT));
+        basePath = buildBasePath();
+    }
+
+    public SecurityDataEnrichServiceClientImpl(String host, int port, String context) {
+        this.host = host;
+        this.port = port;
+        this.context = context;
+        this.basePath = buildBasePath();
+        ClientConfig cc = new DefaultClientConfig();
+        cc.getProperties().put(DefaultClientConfig.PROPERTY_CONNECT_TIMEOUT, 60 * 1000);
+        cc.getProperties().put(DefaultClientConfig.PROPERTY_READ_TIMEOUT, 60 * 1000);
+        cc.getClasses().add(JacksonJsonProvider.class);
+        cc.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND, true);
+        this.client = Client.create(cc);
+        client.addFilter(new com.sun.jersey.api.client.filter.GZIPContentEncodingFilter());
+    }
+
+    private String buildBasePath() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("http://");
+        sb.append(host);
+        sb.append(":");
+        sb.append(port);
+        sb.append(context);
+        return sb.toString();
+    }
+
+    private <T> List<T> list(String path, GenericType<List<T>> type) {
+        WebResource r = client.resource(basePath + path);
+        LOG.info("query URL {}", basePath + path);
+        List<T> ret = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).get(type);
+        return ret;
+    }
+
+    @Override
+    public void close() throws IOException {
+        client.destroy();
+    }
+
+    @Override
+    public Collection<HBaseSensitivityEntity> listHBaseSensitivities() {
+        return list(METADATA_LIST_HBASE_SENSITIVITY_PATH, new GenericType<List<HBaseSensitivityEntity>>() {
+        });
+    }
+
+    @Override
+    public OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h) {
+        WebResource r = client.resource(basePath + METADATA_ADD_HBASE_SENSITIVITY_PATH);
+        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+        return new OpResult();
+    }
+
+    @Override
+    public Collection<HdfsSensitivityEntity> listHdfsSensitivities() {
+        return list(METADATA_LIST_HDFS_SENSITIVITY_PATH, new GenericType<List<HdfsSensitivityEntity>>() {
+        });
+    }
+
+    @Override
+    public OpResult addHdfsSensitivity(Collection<HdfsSensitivityEntity> h) {
+        WebResource r = client.resource(basePath + METADATA_ADD_HDFS_SENSITIVITY_PATH);
+        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+        return new OpResult();
+    }
+
+    @Override
+    public Collection<IPZoneEntity> listIPZones(){
+        return list(METADATA_LIST_IPZONE_PATH, new GenericType<List<IPZoneEntity>>() {
+        });
+    }
+
+    @Override
+    public OpResult addIPZone(Collection<IPZoneEntity> h){
+        WebResource r = client.resource(basePath + METADATA_ADD_IPZONE_PATH);
+        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+        return new OpResult();
+    }
+
+    @Override
+    public Collection<HiveSensitivityEntity> listHiveSensitivities() {
+        return list(METADATA_LIST_HIVE_SENSITIVITY_PATH, new GenericType<List<HiveSensitivityEntity>>() {
+        });
+    }
+
+    @Override
+    public OpResult addHiveSensitivity(Collection<HiveSensitivityEntity> h) {
+        WebResource r = client.resource(basePath + METADATA_ADD_HIVE_SENSITIVITY_PATH);
+        r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON).post(h);
+        return new OpResult();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java
deleted file mode 100644
index 8cd1317..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/AbstractResourceSensitivityPollingJob.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.security.util;
-
-
-import org.apache.eagle.common.config.EagleConfigConstants;
-import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
-import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.service.client.IEagleServiceClient;
-import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
-import org.quartz.JobDataMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-public class AbstractResourceSensitivityPollingJob {
-    private final static Logger LOG = LoggerFactory.getLogger(AbstractResourceSensitivityPollingJob.class);
-
-    protected <R extends TaggedLogAPIEntity> List<R> load(JobDataMap jobDataMap, String service) throws Exception {
-        Map<String, Object> map = (Map<String, Object>) jobDataMap.get(EagleConfigConstants.EAGLE_SERVICE);
-        String eagleServiceHost = (String) map.get(EagleConfigConstants.HOST);
-        Integer eagleServicePort = Integer.parseInt(map.get(EagleConfigConstants.PORT).toString());
-        String username = map.containsKey(EagleConfigConstants.USERNAME) ? (String) map.get(EagleConfigConstants.USERNAME) : null;
-        String password = map.containsKey(EagleConfigConstants.PASSWORD) ? (String) map.get(EagleConfigConstants.PASSWORD) : null;
-
-        // load from eagle database
-        LOG.info("Load resource sensitivity information from eagle service "
-                + eagleServiceHost + ":" + eagleServicePort);
-
-        IEagleServiceClient client = new EagleServiceClientImpl(eagleServiceHost, eagleServicePort, username, password);
-        String query = service + "[]{*}";
-        GenericServiceAPIResponseEntity<R> response =
-                client.search().pageSize(Integer.MAX_VALUE).query(query).send();
-        client.close();
-        if (response.getException() != null)
-            throw new IllegalStateException(response.getException());
-        return response.getObj();
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java
deleted file mode 100644
index 6b7cda4..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataCache.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.util;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.quartz.Job;
-
-public class ExternalDataCache {
-	private Map<Class<? extends Job>, Object> map = new ConcurrentHashMap<Class<? extends Job>, Object>();
-	
-	private static final ExternalDataCache instance = new ExternalDataCache();
-	
-	private ExternalDataCache(){
-	}
-	
-	public static ExternalDataCache getInstance(){
-		return instance;
-	}
-	
-	public Object getJobResult(Class<? extends Job> cls){
-		return map.get(cls);
-	}
-	
-	public void setJobResult(Class<? extends Job> cls, Object jobResult){
-		map.put(cls, jobResult);
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
deleted file mode 100644
index e6af377..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/util/ExternalDataJoiner.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.util;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.eagle.common.config.EagleConfigConstants;
-import org.quartz.Job;
-import org.quartz.JobBuilder;
-import org.quartz.JobDataMap;
-import org.quartz.JobDetail;
-import org.quartz.Scheduler;
-import org.quartz.SimpleScheduleBuilder;
-import org.quartz.Trigger;
-import org.quartz.TriggerBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigValue;
-
-/**
- * spawn Quartz scheduler to poll data from eagle service and invoke custom job to handle
- */
-public class ExternalDataJoiner {
-	private static final Logger LOG = LoggerFactory.getLogger(ExternalDataJoiner.class);
-	
-	private Scheduler sched;
-	private JobDataMap jobDataMap;
-	private Class<? extends Job> jobCls;
-	private String id;
-	private final static String SCHEDULER_NAME = "OuterDataJoiner.scheduler";
-	
-	private static final String DATA_JOIN_POLL_INTERVALSEC = "dataJoinPollIntervalSec";
-	private static final String QUARTZ_GROUP_NAME = "dataJoiner";
-	
-	private final int defaultIntervalSeconds = 60;
-	
-	/**
-	 * constructor with parameters which are passed from customer
-	 * throws Exception while this scheduler can't be brought up
-	 * Some protocols:
-	 * 1. prop should include an entry keyed by "data.join.poll.intervalInSeconds" if you want to customize poll interval. 
-	 * @param prop
-	 */
-	public ExternalDataJoiner(Class<? extends Job> jobCls, Map<String, Object> prop) throws Exception{
-		init(jobCls, prop);
-	}
-	
-	public ExternalDataJoiner(Class<? extends Job> jobCls, Config config, String id) throws Exception{
-		this.id = id;
-		Map<String, Object> map = new HashMap<>();
-        for(Map.Entry<String, ConfigValue> entry : config.getObject("eagleService").entrySet()){
-            map.put(entry.getKey(), entry.getValue().unwrapped());
-        }
-
-		init(jobCls, map);
-	}
-	
-	private void init(Class<? extends Job> jobCls, Map<String, Object> prop) throws Exception{
-		this.jobCls = jobCls;
-		jobDataMap = new JobDataMap(prop);
-		sched = JVMSingleQuartzScheduler.getInstance().getScheduler();
-	}
-	
-	public void start(){
-		// for job
-		String group = String.format("%s.%s.%s.%s", QUARTZ_GROUP_NAME, jobDataMap.getString(EagleConfigConstants.SITE), jobDataMap.getString(EagleConfigConstants.APPLICATION), id);
-		JobDetail job = JobBuilder.newJob(jobCls)
-		     .withIdentity(jobCls.getName() + ".job", group)
-		     .usingJobData(jobDataMap)
-		     .build();
-		
-		// for trigger
-		Object interval = jobDataMap.get(DATA_JOIN_POLL_INTERVALSEC);
-        int dataJoinPollIntervalSec = (interval == null ? defaultIntervalSeconds : Integer.parseInt(interval.toString()));
-		Trigger trigger = TriggerBuilder.newTrigger() 
-			  .withIdentity(jobCls.getName() + ".trigger", group)
-		      .startNow() 
-		      .withSchedule(SimpleScheduleBuilder.simpleSchedule() 
-		          .withIntervalInSeconds(dataJoinPollIntervalSec)
-		          .repeatForever()) 
-		      .build(); 
-		try{
-			sched.scheduleJob(job, trigger);
-		}catch(Exception ex){
-			LOG.error("Can't schedule job " + job.getDescription(), ex);
-		}
-	}
-	
-	public void stop() throws Exception{
-		sched.shutdown();
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
new file mode 100644
index 0000000..2cb1dec
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.hbase;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.service.HBaseSensitivityEntity;
+import org.apache.eagle.security.service.ISecurityDataEnrichServiceClient;
+import org.apache.eagle.security.enrich.AbstractDataEnrichLCM;
+
+import java.util.Collection;
+
+/**
+ * Since 8/16/16.
+ */
+public class HBaseSensitivityDataEnrichLCM extends AbstractDataEnrichLCM<HBaseSensitivityEntity, String> {
+    public HBaseSensitivityDataEnrichLCM(Config config){
+        super(config);
+    }
+    @Override
+    protected Collection<HBaseSensitivityEntity> loadFromService(ISecurityDataEnrichServiceClient client) {
+        return client.listHBaseSensitivities();
+    }
+
+    @Override
+    public String getCacheKey(HBaseSensitivityEntity entity) {
+        return entity.getHbaseResource();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
index cb2fa38..c1005cd 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
@@ -6,27 +6,22 @@
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 package org.apache.eagle.security.hbase;
 
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
 import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
 import backtype.storm.tuple.Fields;
 import backtype.storm.tuple.Tuple;
 import com.typesafe.config.Config;
-import org.apache.eagle.security.entity.HbaseResourceSensitivityAPIEntity;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.apache.eagle.security.util.ExternalDataJoiner;
+import org.apache.eagle.security.service.HBaseSensitivityEntity;
+import org.apache.eagle.security.enrich.AbstractDataEnrichBolt;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,44 +30,22 @@ import java.util.Map;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 
-public class HbaseResourceSensitivityDataJoinBolt extends BaseRichBolt {
+public class HbaseResourceSensitivityDataJoinBolt extends AbstractDataEnrichBolt<HBaseSensitivityEntity, String>  {
     private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityDataJoinBolt.class);
-    private Config config;
-    private OutputCollector collector;
 
     public HbaseResourceSensitivityDataJoinBolt(Config config){
-        this.config = config;
-    }
-
-    @Override
-    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
-        this.collector = collector;
-        // start hbase sensitivity data polling
-        try {
-            ExternalDataJoiner joiner = new ExternalDataJoiner(
-                    HbaseResourceSensitivityPollingJob.class, config, context.getThisComponentId() + "." + context.getThisTaskIndex());
-            joiner.start();
-        } catch(Exception ex){
-            LOG.error("Fail bringing up quartz scheduler.", ex);
-            throw new IllegalStateException(ex);
-        }
+        super(config, new HBaseSensitivityDataEnrichLCM(config));
     }
 
     @Override
-    public void execute(Tuple input) {
+    public void executeWithEnrich(Tuple input, Map<String, HBaseSensitivityEntity> map) {
         try {
-            @SuppressWarnings("unchecked")
             Map<String, Object> event = (Map<String, Object>) input.getValue(0);
-            @SuppressWarnings("unchecked")
-            Map<String, HbaseResourceSensitivityAPIEntity> map =
-                    (Map<String, HbaseResourceSensitivityAPIEntity>) ExternalDataCache
-                            .getInstance()
-                            .getJobResult(HbaseResourceSensitivityPollingJob.class);
             LOG.info(">>>> event: " + event + " >>>> map: " + map);
 
             String resource = (String) event.get("scope");
 
-            HbaseResourceSensitivityAPIEntity sensitivityEntity = null;
+            HBaseSensitivityEntity sensitivityEntity = null;
 
             if (map != null && resource != "") {
                 for (String key : map.keySet()) {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
deleted file mode 100644
index a04f1b2..0000000
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityPollingJob.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Maps;
-import org.apache.eagle.common.config.EagleConfigConstants;
-import org.apache.eagle.security.service.*;
-import org.apache.eagle.security.util.AbstractResourceSensitivityPollingJob;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.quartz.Job;
-import org.quartz.JobDataMap;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.Map;
-
-public class HbaseResourceSensitivityPollingJob extends AbstractResourceSensitivityPollingJob implements Job {
-    private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityPollingJob.class);
-
-    @Override
-    public void execute(JobExecutionContext context)
-            throws JobExecutionException {
-        JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
-        try {
-            Collection<HBaseSensitivityEntity> sensitivityEntities = load(jobDataMap);
-            Map<String, HBaseSensitivityEntity> map = Maps.uniqueIndex(
-                    sensitivityEntities,
-                    new Function<HBaseSensitivityEntity, String>() {
-                        @Override
-                        public String apply(HBaseSensitivityEntity input) {
-                            return input.getHbaseResource();
-                        }
-                    });
-            ExternalDataCache.getInstance().setJobResult(getClass(), map);
-        } catch(Exception ex) {
-        	LOG.error("Fail to load hbase resource sensitivity data", ex);
-        }
-    }
-
-    private Collection<HBaseSensitivityEntity> load(JobDataMap jobDataMap) throws Exception {
-        String eagleServiceHost = (String)jobDataMap.get(EagleConfigConstants.HOST);
-        Integer eagleServicePort = Integer.parseInt(jobDataMap.get(EagleConfigConstants.PORT).toString());
-        String username = jobDataMap.containsKey(EagleConfigConstants.USERNAME) ? (String)jobDataMap.get(EagleConfigConstants.USERNAME) : null;
-        String password = jobDataMap.containsKey(EagleConfigConstants.PASSWORD) ? (String)jobDataMap.get(EagleConfigConstants.PASSWORD) : null;
-
-        // load from eagle database
-        LOG.info("Load hbase resource sensitivity information from eagle service "
-                + eagleServiceHost + ":" + eagleServicePort);
-
-        IMetadataServiceClient client = new MetadataServiceClientImpl(eagleServiceHost, eagleServicePort, "/rest");
-        return client.listHBaseSensitivities();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
index 5bdba3d..3d452b4 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/AbstractHdfsAuditLogApplication.java
@@ -76,11 +76,11 @@ public abstract class AbstractHdfsAuditLogApplication extends StormApplication {
             boltDeclarer.customGrouping("ingest", new CustomPartitionGrouping(createStrategy(config)));
         }
 
-        FileSensitivityDataJoinBolt sensitivityDataJoinBolt = new FileSensitivityDataJoinBolt(config);
+        HdfsSensitivityDataEnrichBolt sensitivityDataJoinBolt = new HdfsSensitivityDataEnrichBolt(config);
         BoltDeclarer sensitivityDataJoinBoltDeclarer = builder.setBolt("sensitivityJoin", sensitivityDataJoinBolt, numOfSensitivityJoinTasks);
         sensitivityDataJoinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
 
-        IPZoneDataJoinBolt ipZoneDataJoinBolt = new IPZoneDataJoinBolt(config);
+        IPZoneDataEnrichBolt ipZoneDataJoinBolt = new IPZoneDataEnrichBolt(config);
         BoltDeclarer ipZoneDataJoinBoltDeclarer = builder.setBolt("ipZoneJoin", ipZoneDataJoinBolt, numOfIPZoneJoinTasks);
         ipZoneDataJoinBoltDeclarer.fieldsGrouping("sensitivityJoin", new Fields("user"));
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java
deleted file mode 100644
index f2582f1..0000000
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/FileSensitivityDataJoinBolt.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.auditlog;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import com.typesafe.config.Config;
-import org.apache.eagle.security.auditlog.timer.FileSensitivityPollingJob;
-import org.apache.eagle.security.auditlog.util.SimplifyPath;
-import org.apache.eagle.security.entity.FileSensitivityAPIEntity;
-import org.apache.eagle.security.util.ExternalDataCache;
-import org.apache.eagle.security.util.ExternalDataJoiner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class FileSensitivityDataJoinBolt extends BaseRichBolt {
-    private static Logger LOG = LoggerFactory.getLogger(FileSensitivityDataJoinBolt.class);
-    private Config config;
-    private OutputCollector collector;
-
-    public FileSensitivityDataJoinBolt(Config config){
-        this.config = config;
-    }
-
-
-    @Override
-    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
-        this.collector = collector;
-        // start hdfs sensitivity data polling
-        try{
-            ExternalDataJoiner joiner = new ExternalDataJoiner(
-                    FileSensitivityPollingJob.class, config, context.getThisComponentId() + "." + context.getThisTaskIndex());
-            joiner.start();
-        }catch(Exception ex){
-            LOG.error("Fail bringing up quartz scheduler", ex);
-            throw new IllegalStateException(ex);
-        }
-    }
-
-    @Override
-    public void execute(Tuple input) {
-        try {
-            Map<String, Object> toBeCopied = (Map<String, Object>) input.getValue(0);
-            Map<String, Object> event = new TreeMap<String, Object>(toBeCopied);
-            Map<String, FileSensitivityAPIEntity> map =
-                    (Map<String, FileSensitivityAPIEntity>) ExternalDataCache.getInstance().
-                            getJobResult(FileSensitivityPollingJob.class);
-            FileSensitivityAPIEntity e = null;
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("Receive map: " + map + "event: " + event);
-            }
-
-            String src = (String) event.get("src");
-            if (map != null && src != null) {
-                String simplifiedPath = new SimplifyPath().build(src);
-                for (String fileDir : map.keySet()) {
-                    Pattern pattern = Pattern.compile(simplifiedPath, Pattern.CASE_INSENSITIVE);
-                    Matcher matcher = pattern.matcher(fileDir);
-                    boolean isMatched = matcher.matches();
-                    if (isMatched) {
-                        e = map.get(fileDir);
-                        break;
-                    }
-                }
-            }
-            event.put("sensitivityType", e == null ? "NA" : e.getSensitivityType());
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("After file sensitivity lookup: " + event);
-            }
-            // LOG.info(">>>> After file sensitivity lookup: " + event);
-            collector.emit(Arrays.asList(event.get("user"), event));
-        }catch(Exception ex){
-            LOG.error("error joining data, ignore it", ex);
-        }finally {
-            collector.ack(input);
-        }
-    }
-
-    @Override
-    public void declareOutputFields(OutputFieldsDeclarer declarer) {
-        declarer.declare(new Fields("user", "message"));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java
new file mode 100644
index 0000000..2031108
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichBolt.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.auditlog;
+
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.apache.eagle.security.auditlog.util.SimplifyPath;
+import org.apache.eagle.security.service.HdfsSensitivityEntity;
+import org.apache.eagle.security.enrich.AbstractDataEnrichBolt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HdfsSensitivityDataEnrichBolt extends AbstractDataEnrichBolt<HdfsSensitivityEntity, String> {
+    private static Logger LOG = LoggerFactory.getLogger(HdfsSensitivityDataEnrichBolt.class);
+
+    public HdfsSensitivityDataEnrichBolt(Config config){
+        super(config, new HdfsSensitivityDataEnrichLCM(config));
+    }
+
+    @Override
+    public void executeWithEnrich(Tuple input, Map<String, HdfsSensitivityEntity> map) {
+        try {
+            Map<String, Object> toBeCopied = (Map<String, Object>) input.getValue(0);
+            Map<String, Object> event = new TreeMap<String, Object>(toBeCopied);
+            HdfsSensitivityEntity e = null;
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Receive map: " + map + "event: " + event);
+            }
+
+            String src = (String) event.get("src");
+            if (map != null && src != null) {
+                String simplifiedPath = new SimplifyPath().build(src);
+                for (String fileDir : map.keySet()) {
+                    Pattern pattern = Pattern.compile(simplifiedPath, Pattern.CASE_INSENSITIVE);
+                    Matcher matcher = pattern.matcher(fileDir);
+                    boolean isMatched = matcher.matches();
+                    if (isMatched) {
+                        e = map.get(fileDir);
+                        break;
+                    }
+                }
+            }
+            event.put("sensitivityType", e == null ? "NA" : e.getSensitivityType());
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("After file sensitivity lookup: " + event);
+            }
+            // LOG.info(">>>> After file sensitivity lookup: " + event);
+            collector.emit(Arrays.asList(event.get("user"), event));
+        }catch(Exception ex){
+            LOG.error("error joining data, ignore it", ex);
+        }finally {
+            collector.ack(input);
+        }
+    }
+
+    @Override
+    public void declareOutputFields(OutputFieldsDeclarer declarer) {
+        declarer.declare(new Fields("user", "message"));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/92ca7c54/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java
new file mode 100644
index 0000000..8d2f22b
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsSensitivityDataEnrichLCM.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.auditlog;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.service.HdfsSensitivityEntity;
+import org.apache.eagle.security.service.ISecurityDataEnrichServiceClient;
+import org.apache.eagle.security.enrich.AbstractDataEnrichLCM;
+
+import java.util.Collection;
+
+/**
+ * Since 8/16/16.
+ */
+public class HdfsSensitivityDataEnrichLCM extends AbstractDataEnrichLCM<HdfsSensitivityEntity, String> {
+    public HdfsSensitivityDataEnrichLCM(Config config){
+        super(config);
+    }
+    @Override
+    protected Collection<HdfsSensitivityEntity> loadFromService(ISecurityDataEnrichServiceClient client) {
+        return client.listHdfsSensitivities();
+    }
+
+    @Override
+    public String getCacheKey(HdfsSensitivityEntity entity) {
+        return entity.getFiledir();
+    }
+}



Mime
View raw message