ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [07/50] [abbrv] incubator-ranger git commit: RANGER-648: implemented cleanup of policy-engine/context-enrichers to reclaim resouces like threads when no longer needed
Date Wed, 28 Oct 2015 15:59:43 GMT
RANGER-648: implemented cleanup of policy-engine/context-enrichers to reclaim resouces like
threads when no longer needed

Signed-off-by: Madhan Neethiraj <madhan@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/a7453154
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/a7453154
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/a7453154

Branch: refs/heads/master
Commit: a7453154663bc2532894dda78ac43b24b9d27ef1
Parents: 0e47fbf
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Mon Sep 7 16:00:16 2015 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Wed Sep 9 17:38:12 2015 -0700

----------------------------------------------------------------------
 .../RangerAbstractContextEnricher.java          |  42 +++-
 .../RangerAdminTagRetriever.java                |  32 +--
 .../contextenricher/RangerContextEnricher.java  |  21 +-
 .../contextenricher/RangerTagEnricher.java      | 195 ++++++++++++++++---
 .../RangerTagFileStoreRetriever.java            |  35 +---
 .../contextenricher/RangerTagReceiver.java      |  28 ---
 .../contextenricher/RangerTagRetriever.java     |  29 +--
 .../plugin/policyengine/RangerPolicyEngine.java |   5 +
 .../policyengine/RangerPolicyEngineImpl.java    |  61 +++++-
 .../policyengine/RangerPolicyRepository.java    |   6 +-
 .../ranger/plugin/service/RangerBasePlugin.java |  16 +-
 11 files changed, 342 insertions(+), 128 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
index f869d58..c70fdb4 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
@@ -40,7 +40,7 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri
 	protected RangerServiceDef serviceDef;
 
 	@Override
-	public void setContextEnricherDef(RangerContextEnricherDef enricherDef) {
+	public void setEnricherDef(RangerContextEnricherDef enricherDef) {
 		this.enricherDef = enricherDef;
 	}
 	
@@ -56,12 +56,12 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri
 	}
 
 	@Override
-	public void setContextServiceName(String serviceName) {
+	public void setServiceName(String serviceName) {
 		this.serviceName = serviceName;
 	}
 
 	@Override
-	public void setContextServiceDef(RangerServiceDef serviceDef) {
+	public void setServiceDef(RangerServiceDef serviceDef) {
 		this.serviceDef = serviceDef;
 	}
 
@@ -70,6 +70,42 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri
 		this.appId = appId;
 	}
 
+
+	@Override
+	public RangerContextEnricherDef getEnricherDef() {
+		return enricherDef;
+	}
+
+	@Override
+	public String getServiceName() {
+		return serviceName;
+	}
+
+	@Override
+	public RangerServiceDef getServiceDef() {
+		return serviceDef;
+	}
+
+	@Override
+	public String getAppId() {
+		return appId;
+	}
+
+
+	@Override
+	public String getName() {
+		return enricherDef == null ? null : enricherDef.getName();
+	}
+
+	@Override
+	public boolean preCleanup() {
+		return true;
+	}
+
+	@Override
+	public void cleanup() {
+	}
+
 	public String getOption(String name) {
 		String ret = null;
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
index 9db8f0a..7c5b378 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
@@ -36,7 +36,8 @@ public class RangerAdminTagRetriever extends RangerTagRetriever {
 
 	@Override
 	public void init(Map<String, String> options) {
-		if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)
&& tagReceiver != null) {
+
+		if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId))
{
 			String propertyPrefix    = "ranger.plugin." + serviceDef.getName();
 
 			adminClient = RangerBasePlugin.createAdminClient(serviceName, appId, propertyPrefix);
@@ -47,37 +48,24 @@ public class RangerAdminTagRetriever extends RangerTagRetriever {
 	}
 
 	@Override
-	public void retrieveTags() throws InterruptedException {
+	public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException {
+
+		ServiceTags serviceTags = null;
 
-		if (adminClient != null && tagReceiver != null) {
-			ServiceTags serviceTags = null;
+		if (adminClient != null) {
 			try {
 				serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion);
-			}
-			catch (InterruptedException interruptedException) {
+			} catch (InterruptedException interruptedException) {
 				LOG.error("Tag-retriever thread was interrupted");
 				throw interruptedException;
-			}
-			catch (ClosedByInterruptException closedByInterruptException) {
+			} catch (ClosedByInterruptException closedByInterruptException) {
 				LOG.error("Tag-retriever thread was interrupted while blocked on I/O");
 				throw new InterruptedException();
-			}
-			catch (Exception exception) {
+			} catch (Exception exception) {
 				LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources, exception=",
exception);
 			}
-
-			if (serviceTags != null) {
-				tagReceiver.setServiceTags(serviceTags);
-				LOG.info("RangerAdminTagRetriever.retrieveTags() - Updated tags-cache to new version
of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion());
-				setLastKnownVersion(serviceTags.getTagVersion());
-			} else {
-				if (LOG.isDebugEnabled()) {
-					LOG.debug("RangerAdminTagRetriever.retrieveTags() - No need to update tags-cache. lastKnownVersion="
+ lastKnownVersion);
-				}
-			}
-		} else {
-			LOG.error("RangerAdminTagRetriever.retrieveTags() - No admin client to get tags from or
no tag receiver to update tag-cache");
 		}
+		return serviceTags;
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
index 10fed69..c88cbf1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
@@ -25,14 +25,24 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerContextEnricherDef;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
 
 public interface RangerContextEnricher {
-	void setContextEnricherDef(RangerContextEnricherDef enricherDef);
+	void setEnricherDef(RangerContextEnricherDef enricherDef);
 
-	void setContextServiceName(String serviceName);
+	void setServiceName(String serviceName);
 
-	void setContextServiceDef(RangerServiceDef serviceDef);
+	void setServiceDef(RangerServiceDef serviceDef);
 
 	void setAppId(String appId);
 
+	RangerContextEnricherDef getEnricherDef();
+
+	String getServiceName();
+
+	RangerServiceDef getServiceDef();
+
+	String getAppId();
+
+	String getName();
+
 	//void setContextComponentServiceName(String componentServiceName);
 
 	//void setContextComponentServiceDef(RangerServiceDef componentServiceDef);
@@ -40,4 +50,9 @@ public interface RangerContextEnricher {
 	void init();
 
 	void enrich(RangerAccessRequest request);
+
+	boolean preCleanup();
+
+	void cleanup();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 37732a6..6388c78 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -19,11 +19,14 @@
 
 package org.apache.ranger.plugin.contextenricher;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
 import org.apache.ranger.plugin.model.RangerServiceResource;
 import org.apache.ranger.plugin.model.RangerTag;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
@@ -32,11 +35,12 @@ import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourc
 import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
 import org.apache.ranger.plugin.util.ServiceTags;
 
+import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-public class RangerTagEnricher extends RangerAbstractContextEnricher implements RangerTagReceiver
{
+public class RangerTagEnricher extends RangerAbstractContextEnricher {
 	private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class);
 
 	public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval";
@@ -67,8 +71,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
 
 		if (StringUtils.isNotBlank(tagRetrieverClassName)) {
 
-			cleanup();
-
 			try {
 				@SuppressWarnings("unchecked")
 				Class<RangerTagRetriever> tagRetriverClass = (Class<RangerTagRetriever>)
Class.forName(tagRetrieverClassName);
@@ -86,21 +88,26 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
 			}
 
 			if (tagRetriever != null) {
+				String propertyPrefix    = "ranger.plugin." + serviceDef.getName();
+				String cacheDir          = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.cache.dir");
+				String cacheFilename = String.format("%s_%s_tag.json", appId, serviceName);
+				cacheFilename = cacheFilename.replace(File.separatorChar,  '_');
+				cacheFilename = cacheFilename.replace(File.pathSeparatorChar,  '_');
+
+				String cacheFile = cacheDir == null ? null : (cacheDir + File.separator + cacheFilename);
 				tagRetriever.setServiceName(serviceName);
 				tagRetriever.setServiceDef(serviceDef);
 				tagRetriever.setAppId(appId);
-				tagRetriever.setLastKnownVersion(lastKnownVersion);
-				tagRetriever.setTagReceiver(this);
 				tagRetriever.init(enricherDef.getEnricherOptions());
 
+				tagRefresher = new RangerTagRefresher(tagRetriever, this, lastKnownVersion, cacheFile,
pollingIntervalMs);
+
 				try {
-					tagRetriever.retrieveTags();
-				} catch (Exception exception) {
-					// Ignore
+					tagRefresher.populateTags();
+				} catch (Throwable exception) {
+					LOG.error("Exception when retrieving tag for the first time for this enricher", exception);
 				}
 
-				tagRefresher = new RangerTagRefresher(tagRetriever, pollingIntervalMs);
-
 				tagRefresher.startRefresher();
 			}
 		} else {
@@ -112,14 +119,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher
implements
 		}
 	}
 
-	public void cleanup() {
-
-		if (tagRefresher != null) {
-			tagRefresher.cleanup();
-			tagRefresher = null;
-		}
-	}
-
 	@Override
 	public void enrich(RangerAccessRequest request) {
 		if (LOG.isDebugEnabled()) {
@@ -147,7 +146,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
 		}
 	}
 
-	@Override
 	public void setServiceTags(final ServiceTags serviceTags) {
 		this.serviceTags = serviceTags;
 		this.lastKnownVersion = serviceTags.getTagVersion();
@@ -181,6 +179,25 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher
implements
 
 	}
 
+	@Override
+	public boolean preCleanup() {
+		boolean ret = true;
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> RangerTagEnricher.preCleanup()");
+		}
+
+		if (tagRefresher != null) {
+			tagRefresher.cleanup();
+			tagRefresher = null;
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== RangerTagEnricher.preCleanup() : result=" + ret);
+		}
+		return ret;
+	}
+
 	private List<RangerTag> findMatchingTags(final RangerAccessResource resource, final
List<RangerServiceResourceMatcher> resourceMatchers) {
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("==> RangerTagEnricher.findMatchingTags(" + resource + ")");
@@ -252,16 +269,30 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher
implements
 		private static final Log LOG = LogFactory.getLog(RangerTagRefresher.class);
 
 		private final RangerTagRetriever tagRetriever;
+		private final RangerTagEnricher tagEnricher;
+		private long lastKnownVersion = -1L;
 
 		private final long pollingIntervalMs;
+		private final String cacheFile;
+		private boolean hasProvidedTagsToReceiver = false;
+		private Gson gson;
+
 
 		final long getPollingIntervalMs() {
 			return pollingIntervalMs;
 		}
 
-		RangerTagRefresher(RangerTagRetriever tagRetriever, long pollingIntervalMs) {
+		RangerTagRefresher(RangerTagRetriever tagRetriever, RangerTagEnricher tagEnricher, long
lastKnownVersion, String cacheFile, long pollingIntervalMs) {
 			this.tagRetriever = tagRetriever;
+			this.tagEnricher = tagEnricher;
+			this.lastKnownVersion = lastKnownVersion;
+			this.cacheFile = cacheFile;
 			this.pollingIntervalMs = pollingIntervalMs;
+			try {
+				gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
+			} catch(Throwable excp) {
+				LOG.fatal("failed to create GsonBuilder object", excp);
+			}
 		}
 
 		@Override
@@ -275,7 +306,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
 
 				try {
 
-					tagRetriever.retrieveTags();
+					populateTags();
 
 					if (pollingIntervalMs > 0) {
 						Thread.sleep(pollingIntervalMs);
@@ -293,8 +324,46 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher
implements
 			}
 		}
 
+		private void populateTags() throws InterruptedException {
+
+			if (tagEnricher != null) {
+				ServiceTags serviceTags = null;
+
+				serviceTags = tagRetriever.retrieveTags(lastKnownVersion);
+
+				if (serviceTags == null) {
+					if (!hasProvidedTagsToReceiver) {
+						serviceTags = loadFromCache();
+					}
+				} else {
+					saveToCache(serviceTags);
+				}
+
+				if (serviceTags != null) {
+					tagEnricher.setServiceTags(serviceTags);
+					LOG.info("RangerTagRefresher.populateTags() - Updated tags-cache to new version of tags,
lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion());
+					lastKnownVersion = serviceTags.getTagVersion();
+					hasProvidedTagsToReceiver = true;
+				} else {
+					if (LOG.isDebugEnabled()) {
+						LOG.debug("RangerTagRefresher.populateTags() - No need to update tags-cache. lastKnownVersion="
+ lastKnownVersion);
+					}
+				}
+			} else {
+				LOG.error("RangerTagRefresher.populateTags() - no tag receiver to update tag-cache");
+			}
+		}
+
 		void cleanup() {
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("==> RangerTagRefresher.cleanup()");
+			}
+
 			stopRefresher();
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("<== RangerTagRefresher.cleanup()");
+			}
 		}
 
 		final void startRefresher() {
@@ -317,6 +386,88 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher
implements
 				}
 			}
 		}
-	}
 
+
+		final ServiceTags loadFromCache() {
+			ServiceTags serviceTags = null;
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()");
+			}
+
+			File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile);
+
+			if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) {
+				Reader reader = null;
+
+				try {
+					reader = new FileReader(cacheFile);
+
+					serviceTags = gson.fromJson(reader, ServiceTags.class);
+
+					if (serviceTags != null) {
+						if (!StringUtils.equals(tagEnricher.getServiceName(), serviceTags.getServiceName()))
{
+							LOG.warn("ignoring unexpected serviceName '" + serviceTags.getServiceName() + "' in
cache file '" + cacheFile.getAbsolutePath() + "'");
+
+							serviceTags.setServiceName(tagEnricher.getServiceName());
+						}
+					}
+				} catch (Exception excp) {
+					LOG.error("failed to load service-tags from cache file " + cacheFile.getAbsolutePath(),
excp);
+				} finally {
+					if (reader != null) {
+						try {
+							reader.close();
+						} catch (Exception excp) {
+							LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(),
excp);
+						}
+					}
+				}
+			} else {
+				LOG.warn("cache file does not exist or not readble '" + (cacheFile == null ? null : cacheFile.getAbsolutePath())
+ "'");
+			}
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()");
+			}
+
+			return serviceTags;
+		}
+
+		final void saveToCache(ServiceTags serviceTags) {
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()");
+			}
+
+			if (serviceTags != null) {
+				File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile);
+
+				if (cacheFile != null) {
+					Writer writer = null;
+
+					try {
+						writer = new FileWriter(cacheFile);
+
+						gson.toJson(serviceTags, writer);
+					} catch (Exception excp) {
+						LOG.error("failed to save service-tags to cache file '" + cacheFile.getAbsolutePath()
+ "'", excp);
+					} finally {
+						if (writer != null) {
+							try {
+								writer.close();
+							} catch (Exception excp) {
+								LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath()
+ "'", excp);
+							}
+						}
+					}
+				}
+			} else {
+				LOG.info("service-tags is null. Nothing to save in cache");
+			}
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()");
+			}
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
index 0259bdf..1ee00d3 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
@@ -34,12 +34,10 @@ public class RangerTagFileStoreRetriever extends RangerTagRetriever {
 
 	private TagStore tagStore;
 
-	public RangerTagFileStoreRetriever() {
-	}
-
 	@Override
 	public void init(Map<String, String> options) {
-		if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)
&& tagReceiver != null) {
+
+		if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId))
{
 
 			tagStore = TagFileStore.getInstance();
 
@@ -49,37 +47,24 @@ public class RangerTagFileStoreRetriever extends RangerTagRetriever {
 	}
 
 	@Override
-	public void retrieveTags() throws InterruptedException {
+	public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException {
 
-		if (tagStore != null && tagReceiver != null) {
-			ServiceTags serviceTags = null;
+		ServiceTags serviceTags = null;
+
+		if (tagStore != null) {
 			try {
 				serviceTags = tagStore.getServiceTagsIfUpdated(serviceName, lastKnownVersion);
-			}
-			catch (InterruptedException interruptedException) {
+			} catch (InterruptedException interruptedException) {
 				LOG.error("Tag-retriever thread was interrupted");
 				throw interruptedException;
-			}
-			catch (ClosedByInterruptException closedByInterruptException) {
+			} catch (ClosedByInterruptException closedByInterruptException) {
 				LOG.error("Tag-retriever thread was interrupted while blocked on I/O");
 				throw new InterruptedException();
-			}
-			catch (Exception exception) {
+			} catch (Exception exception) {
 				LOG.error("RangerTagFileStoreRetriever.retrieveTags() - Error retrieving resources, exception=",
exception);
 			}
-
-			if (serviceTags != null) {
-				tagReceiver.setServiceTags(serviceTags);
-				LOG.info("RangerTagFileStoreRetriever.retrieveTags() - Updated tags-cache to new version
of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion());
-				setLastKnownVersion(serviceTags.getTagVersion());
-			} else {
-				if (LOG.isDebugEnabled()) {
-					LOG.debug("RangerTagFileStoreRetriever.retrieveTags() - No need to update tags-cache.
lastKnownVersion=" + lastKnownVersion);
-				}
-			}
-		} else {
-			LOG.error("RangerTagFileStoreRetriever.retrieveTags() - No tag-store to get tags from
or no tag receiver to update tag-cache...");
 		}
+		return serviceTags;
 	}
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java
deleted file mode 100644
index 47db707..0000000
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ranger.plugin.contextenricher;
-
-import org.apache.ranger.plugin.util.ServiceTags;
-
-import java.util.List;
-
-public interface RangerTagReceiver {
-	void setServiceTags(final ServiceTags serviceTags);
-}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
index 4967cbe..374e5c1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
@@ -19,21 +19,27 @@
 
 package org.apache.ranger.plugin.contextenricher;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.util.ServiceTags;
 
+import java.io.*;
 import java.util.Map;
 
 public abstract class RangerTagRetriever {
+	private static final Log LOG = LogFactory.getLog(RangerTagRetriever.class);
 
 	protected String serviceName;
 	protected RangerServiceDef serviceDef;
 	protected String appId;
-	protected long lastKnownVersion;
-	protected RangerTagReceiver tagReceiver;
 
 	public abstract void init(Map<String, String> options);
 
-	public abstract void retrieveTags() throws InterruptedException;
+	public abstract ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException;
 
 	public String getServiceName() {
 		return serviceName;
@@ -58,21 +64,4 @@ public abstract class RangerTagRetriever {
 	public void setAppId(String appId) {
 		this.appId = appId;
 	}
-
-	public long getLastKnownVersion() {
-		return lastKnownVersion;
-	}
-
-	public void setLastKnownVersion(long lastKnownVersion) {
-		this.lastKnownVersion = lastKnownVersion;
-	}
-
-	public RangerTagReceiver getTagReceiver() {
-		return tagReceiver;
-	}
-
-	public void setTagReceiver(RangerTagReceiver tagReceiver) {
-		this.tagReceiver = tagReceiver;
-	}
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
index 055a2db..29080b7 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
@@ -57,4 +57,9 @@ public interface RangerPolicyEngine {
 	RangerPolicy getExactMatchPolicy(RangerAccessResource resource);
 
 	List<RangerPolicy> getAllowedPolicies(String user, Set<String> userGroups, String
accessType);
+
+	boolean preCleanup();
+
+	void cleanup();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index 3cccde0..5d1140b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -40,7 +40,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 	private final RangerPolicyRepository policyRepository;
 	private final RangerPolicyRepository tagPolicyRepository;
 	
-	private final List<RangerContextEnricher> allContextEnrichers;
+	private List<RangerContextEnricher> allContextEnrichers;
 
 	public RangerPolicyEngineImpl(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions
options) {
 		if (LOG.isDebugEnabled()) {
@@ -484,6 +484,65 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 	}
 
 	@Override
+	public boolean preCleanup() {
+
+		boolean ret = true;
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> RangerPolicyEngineImpl.preCleanup()");
+		}
+
+		if (CollectionUtils.isNotEmpty(allContextEnrichers)) {
+			for (RangerContextEnricher contextEnricher : allContextEnrichers) {
+				boolean notReadyForCleanup = contextEnricher.preCleanup();
+				if (!notReadyForCleanup) {
+					if (LOG.isDebugEnabled()) {
+						LOG.debug("contextEnricher.preCleanup() failed for contextEnricher=" + contextEnricher.getName());
+					}
+					ret = false;
+				}
+			}
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== RangerPolicyEngineImpl.preCleanup() : result=" + ret);
+		}
+
+		return ret;
+	}
+
+	@Override
+	public void cleanup() {
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> RangerPolicyEngineImpl.cleanup()");
+		}
+
+		preCleanup();
+
+		if (CollectionUtils.isNotEmpty(allContextEnrichers)) {
+			for (RangerContextEnricher contextEnricher : allContextEnrichers) {
+				contextEnricher.cleanup();
+			}
+		}
+
+		this.allContextEnrichers = null;
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== RangerPolicyEngineImpl.cleanup()");
+		}
+	}
+
+	@Override
+	protected void finalize() throws Throwable {
+		try {
+			cleanup();
+		}
+		finally {
+			super.finalize();
+		}
+	}
+
+	@Override
 	public String toString( ) {
 		StringBuilder sb = new StringBuilder();
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 8519860..37b1ea6 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -336,9 +336,9 @@ public class RangerPolicyRepository {
         }
 
         if(ret != null) {
-        	ret.setContextEnricherDef(enricherDef);
-            ret.setContextServiceName(componentServiceName);
-            ret.setContextServiceDef(componentServiceDef);
+            ret.setEnricherDef(enricherDef);
+            ret.setServiceName(componentServiceName);
+            ret.setServiceDef(componentServiceDef);
             ret.setAppId(appId);
             ret.init();
         }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index c857484..9d9987b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.admin.client.RangerAdminClient;
 import org.apache.ranger.admin.client.RangerAdminRESTClient;
 import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.plugin.model.RangerPolicy;
 import org.apache.ranger.plugin.model.RangerServiceDef;
 import org.apache.ranger.plugin.policyengine.*;
 import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
@@ -107,21 +108,34 @@ public class RangerBasePlugin {
 	}
 
 	public void setPolicies(ServicePolicies policies) {
+		RangerPolicyEngine oldPolicyEngine = this.policyEngine;
+
 		RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(appId, policies, policyEngineOptions);
 
 		this.policyEngine = policyEngine;
+
+		if (oldPolicyEngine != null && !oldPolicyEngine.preCleanup()) {
+			LOG.error("preCleanup() failed on the previous policy engine instance !!");
+		}
 	}
 
 	public void cleanup() {
+
 		PolicyRefresher refresher = this.refresher;
 
+		RangerPolicyEngine policyEngine = this.policyEngine;
+
 		this.serviceName  = null;
 		this.policyEngine = null;
 		this.refresher    = null;
 
-		if(refresher != null) {
+		if (refresher != null) {
 			refresher.stopRefresher();
 		}
+
+		if (policyEngine != null) {
+			policyEngine.cleanup();
+		}
 	}
 
 	public void setResultProcessor(RangerAccessResultProcessor resultProcessor) {


Mime
View raw message