ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ab...@apache.org
Subject [1/2] ranger git commit: RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin
Date Thu, 27 Sep 2018 01:53:00 GMT
Repository: ranger
Updated Branches:
  refs/heads/ranger-0.7 475b5290a -> dddcf0155


RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via
REST API in Apache Ranger admin


Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/320169a8
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/320169a8
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/320169a8

Branch: refs/heads/ranger-0.7
Commit: 320169a8a796d2f7bf10a45fff4a39fcb80cec9a
Parents: 475b529
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Tue Jul 31 15:15:33 2018 -0700
Committer: Abhay Kulkarni <akulkarni@hortonworks.com>
Committed: Wed Sep 26 18:38:05 2018 -0700

----------------------------------------------------------------------
 .../ranger/biz/RangerPolicyRetriever.java       | 83 ++++++++++++++--
 .../apache/ranger/biz/RangerTagDBRetriever.java | 99 ++++++++++++++++++--
 .../org/apache/ranger/biz/ServiceDBStore.java   |  2 +-
 .../java/org/apache/ranger/biz/TagDBStore.java  |  8 +-
 4 files changed, 175 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index 1b6f440..e867cf4 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -43,18 +43,36 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem;
 import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator;
 import org.apache.ranger.plugin.util.RangerPerfTracer;
-
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
 
 public class RangerPolicyRetriever {
 	static final Log LOG      = LogFactory.getLog(RangerPolicyRetriever.class);
 	static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever");
 
-	final RangerDaoManager daoMgr;
-	final LookupCache      lookupCache;
+	private final RangerDaoManager  daoMgr;
+	private final LookupCache       lookupCache = new LookupCache();
+
+	private final PlatformTransactionManager  txManager;
+	private final TransactionTemplate         txTemplate;
+
+	public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager)
{
+		this.daoMgr     = daoMgr;
+		this.txManager  = txManager;
+		if (this.txManager != null) {
+			this.txTemplate = new TransactionTemplate(this.txManager);
+			this.txTemplate.setReadOnly(true);
+		} else {
+			this.txTemplate = null;
+		}
+	}
 
 	public RangerPolicyRetriever(RangerDaoManager daoMgr) {
 		this.daoMgr      = daoMgr;
-		this.lookupCache = new LookupCache();
+		this.txManager   = null;
+		this.txTemplate  = null;
 	}
 
 	public List<RangerPolicy> getServicePolicies(Long serviceId) {
@@ -93,7 +111,41 @@ public class RangerPolicyRetriever {
 		return ret;
 	}
 
-	public List<RangerPolicy> getServicePolicies(XXService xService) {
+	private class PolicyLoaderThread extends Thread {
+		final TransactionTemplate txTemplate;
+		final XXService           xService;
+		List<RangerPolicy>  policies;
+
+		PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+			this.txTemplate = txTemplate;
+			this.xService   = xService;
+		}
+
+		public List<RangerPolicy> getPolicies() { return policies; }
+
+		@Override
+		public void run() {
+			try {
+				policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>()
{
+					@Override
+					public List<RangerPolicy> doInTransaction(TransactionStatus status) {
+						try {
+							RetrieverContext ctx = new RetrieverContext(xService);
+							return ctx.getAllPolicies();
+						} catch (Exception ex) {
+							LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for
service:[" + xService.getName() + "] in a new transaction", ex);
+							status.setRollbackOnly();
+							return null;
+						}
+					}
+				});
+			} catch (Throwable ex) {
+				LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:["
+ xService.getName() + "] in a new transaction", ex);
+			}
+		}
+	}
+
+	public List<RangerPolicy> getServicePolicies(final XXService xService) {
 		String serviceName = xService == null ? null : xService.getName();
 		Long   serviceId   = xService == null ? null : xService.getId();
 
@@ -109,9 +161,26 @@ public class RangerPolicyRetriever {
 		}
 
 		if(xService != null) {
-			RetrieverContext ctx = new RetrieverContext(xService);
+			if (txTemplate == null) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction");
+				}
+				RetrieverContext ctx = new RetrieverContext(xService);
+				ret = ctx.getAllPolicies();
+			} else {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Retrieving policies in a new, read-only transaction");
+				}
 
-			ret = ctx.getAllPolicies();
+				PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService);
+				t.start();
+				try {
+					t.join();
+					ret = t.getPolicies();
+				} catch (InterruptedException ie) {
+					LOG.error("Failed to retrieve policies in a new, read-only thread.", ie);
+				}
+			}
 		} else {
 			if(LOG.isDebugEnabled()) {
 				LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid
parameter");

http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 52c1288..26d8d37 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -35,6 +35,10 @@ import org.apache.ranger.entity.*;
 import org.apache.ranger.plugin.model.*;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
 import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
 
 
 public class RangerTagDBRetriever {
@@ -46,6 +50,9 @@ public class RangerTagDBRetriever {
 	private final XXService xService;
 	private final LookupCache lookupCache;
 
+	private final PlatformTransactionManager  txManager;
+	private final TransactionTemplate         txTemplate;
+
 	private List<RangerServiceResource> serviceResources;
 	private Map<Long, RangerTagDef> tagDefs;
 	private Map<Long, RangerTag> tags;
@@ -53,8 +60,15 @@ public class RangerTagDBRetriever {
 
 	private boolean filterForServicePlugin;
 
-	public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) {
+	public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager
txManager, final XXService xService) {
 		this.daoMgr = daoMgr;
+		this.txManager = txManager;
+		if (this.txManager != null) {
+			this.txTemplate = new TransactionTemplate(this.txManager);
+			this.txTemplate.setReadOnly(true);
+		} else {
+			this.txTemplate = null;
+		}
 		this.xService = xService;
 		this.lookupCache = new LookupCache();
 
@@ -68,14 +82,27 @@ public class RangerTagDBRetriever {
 			}
 
 			filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN,
false);
-			TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
-			TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
-			TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
 
-			serviceResources = serviceResourceContext.getAllServiceResources();
-			tagDefs = tagDefContext.getAllTagDefs();
-			tags = tagContext.getAllTags();
-			tagResourceMaps = getAllTagResourceMaps();
+			if (this.txTemplate == null) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Load Tags in the same thread and using an existing transaction");
+				}
+				if (initializeTagCache(xService) == false) {
+					LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread
and using an existing transaction");
+				}
+			} else {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Load Tags in a separate thread and using a new transaction");
+				}
+
+				TagLoaderThread t = new TagLoaderThread(txTemplate, xService);
+				t.start();
+				try {
+					t.join();
+				} catch (InterruptedException ie) {
+					LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie);
+				}
+			}
 
 			RangerPerfTracer.log(perf);
 
@@ -98,6 +125,30 @@ public class RangerTagDBRetriever {
 		return tags;
 	}
 
+	private boolean initializeTagCache(XXService xService) {
+		boolean ret;
+		try {
+			TagRetrieverServiceResourceContext  serviceResourceContext  = new TagRetrieverServiceResourceContext(xService);
+			TagRetrieverTagDefContext           tagDefContext           = new TagRetrieverTagDefContext(xService);
+			TagRetrieverTagContext              tagContext              = new TagRetrieverTagContext(xService);
+
+			serviceResources    = serviceResourceContext.getAllServiceResources();
+			tagDefs             = tagDefContext.getAllTagDefs();
+			tags                = tagContext.getAllTags();
+
+			tagResourceMaps     = getAllTagResourceMaps();
+
+			ret = true;
+		} catch (Exception ex) {
+			LOG.error("Failed to get tags for service:[" + xService.getName() + "]");
+			serviceResources    = null;
+			tagDefs             = null;
+			tags                = null;
+			tagResourceMaps     = null;
+			ret = false;
+		}
+		return ret;
+	}
 	private List<RangerTagResourceMap> getAllTagResourceMaps() {
 
 		List<XXTagResourceMap> xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId())
: daoMgr.getXXTagResourceMap().findByServiceId(xService.getId());
@@ -198,6 +249,38 @@ public class RangerTagDBRetriever {
 		}
 	}
 
+	private class TagLoaderThread extends Thread {
+		final TransactionTemplate txTemplate;
+		final XXService           xService;
+
+		TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+			this.txTemplate = txTemplate;
+			this.xService   = xService;
+		}
+
+		@Override
+		public void run() {
+			try {
+				 Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+					@Override
+					public Boolean doInTransaction(TransactionStatus status) {
+						boolean ret = initializeTagCache(xService);
+						if (!ret) {
+							status.setRollbackOnly();
+							LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction");
+						}
+						return ret;
+					}
+				});
+				 if (LOG.isDebugEnabled()) {
+				 	LOG.debug("transaction result:[" + result +"]");
+				 }
+			} catch (Throwable ex) {
+				LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction",
ex);
+			}
+		}
+	}
+
 	private class TagRetrieverServiceResourceContext {
 
 		final XXService service;

http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index ed6ddac..64cf043 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -2407,7 +2407,7 @@ public class ServiceDBStore extends AbstractServiceStore {
 			LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")");
 		}
 
-		RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr);
+		RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager);
 
 		List<RangerPolicy> ret = policyRetriever.getServicePolicies(service);
 

http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index a115bb7..3234be6 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -62,7 +62,9 @@ import org.apache.ranger.service.RangerTagResourceMapService;
 import org.apache.ranger.service.RangerTagService;
 import org.apache.ranger.service.RangerServiceResourceService;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
 
 import javax.annotation.PostConstruct;
 import javax.servlet.http.HttpServletResponse;
@@ -87,6 +89,10 @@ public class TagDBStore extends AbstractTagStore {
 	RangerDaoManager daoManager;
 
 	@Autowired
+	@Qualifier(value = "transactionManager")
+	PlatformTransactionManager txManager;
+
+	@Autowired
 	RESTErrorUtil errorUtil;
 
 	@Autowired
@@ -985,7 +991,7 @@ public class TagDBStore extends AbstractTagStore {
 			throw new Exception("service-def does not exist. id=" + xxService.getType());
 		}
 
-		RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService);
+		RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService);
 
 		Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs();
 		Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();


Mime
View raw message