ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From me...@apache.org
Subject [6/9] ranger git commit: RANGER-2203, RANGER-2219: Review and update database schema for ranger policies and tag objects to minimize database queries/updates
Date Mon, 22 Oct 2018 06:42:41 GMT
http://git-wip-us.apache.org/repos/asf/ranger/blob/3af1f59d/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index c26f0a5..36a7b4b 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -27,31 +27,29 @@ import java.util.Map;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.authorization.utils.JsonUtils;
 import org.apache.ranger.authorization.utils.StringUtil;
 import org.apache.ranger.db.RangerDaoManager;
-import org.apache.ranger.entity.*;
+import org.apache.ranger.entity.XXPolicy;
+import org.apache.ranger.entity.XXPolicyLabel;
+import org.apache.ranger.entity.XXPolicyLabelMap;
+import org.apache.ranger.entity.XXPortalUser;
+import org.apache.ranger.entity.XXService;
 import org.apache.ranger.plugin.model.RangerPolicy;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition;
-import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo;
-import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
-import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem;
-import org.apache.ranger.plugin.model.RangerValiditySchedule;
-import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator;
 import org.apache.ranger.plugin.util.RangerPerfTracer;
-import org.apache.ranger.service.RangerPolicyService;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.TransactionStatus;
 import org.springframework.transaction.support.TransactionCallback;
 import org.springframework.transaction.support.TransactionTemplate;
 
+
 public class RangerPolicyRetriever {
 	static final Log LOG      = LogFactory.getLog(RangerPolicyRetriever.class);
 	static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever");
@@ -303,35 +301,15 @@ public class RangerPolicyRetriever {
 	}
 
 	class LookupCache {
-		final Map<Long, String> userNames       = new HashMap<Long, String>();
-		final Map<Long, String> userScreenNames = new HashMap<Long, String>();
-		final Map<Long, String> groupNames      = new HashMap<Long, String>();
-		final Map<Long, String> accessTypes     = new HashMap<Long, String>();
-		final Map<Long, String> conditions      = new HashMap<Long, String>();
-		final Map<Long, String> resourceDefs    = new HashMap<Long, String>();
-		final Map<Long, String> dataMasks       = new HashMap<Long, String>();
+		final Map<Long, String>              userScreenNames            = new HashMap<Long, String>();
+		final Map<Long, Map<String, String>> groupMappingsPerPolicy     = new HashMap<>();
+		final Map<Long, Map<String, String>> userMappingsPerPolicy      = new HashMap<>();
+		final Map<Long, Map<String, String>> accessMappingsPerPolicy    = new HashMap<>();
+		final Map<Long, Map<String, String>> resourceMappingsPerPolicy  = new HashMap<>();
+		final Map<Long, Map<String, String>> dataMaskMappingsPerPolicy  = new HashMap<>();
+		final Map<Long, Map<String, String>> conditionMappingsPerPolicy = new HashMap<>();
 		final Map<Long, String> policyLabels    = new HashMap<Long, String>();
 
-		String getUserName(Long userId) {
-			String ret = null;
-
-			if(userId != null) {
-				ret = userNames.get(userId);
-
-				if(ret == null) {
-					XXUser user = daoMgr.getXXUser().getById(userId);
-
-					if(user != null) {
-						ret = user.getName(); // Name is `loginId`
-
-						userNames.put(userId,  ret);
-					}
-				}
-			}
-
-			return ret;
-		}
-
 		String getPolicyLabelName(Long policyLabelId) {
 			String ret = null;
 
@@ -386,233 +364,141 @@ public class RangerPolicyRetriever {
 			return ret;
 		}
 
-		String getGroupName(Long groupId) {
-			String ret = null;
-
-			if(groupId != null) {
-				ret = groupNames.get(groupId);
+		void setNameMapping(Map<Long, Map<String, String>> nameMappingContainer, List<PolicyTextNameMap> nameMappings) {
+			nameMappingContainer.clear();
 
-				if(ret == null) {
-					XXGroup group = daoMgr.getXXGroup().getById(groupId);
+			for (PolicyTextNameMap nameMapping : nameMappings) {
+				Map<String, String> policyNameMap = nameMappingContainer.get(nameMapping.policyId);
 
-					if(group != null) {
-						ret = group.getName();
+				if (policyNameMap == null) {
+					policyNameMap = new HashMap<>();
 
-						groupNames.put(groupId,  ret);
-					}
+					nameMappingContainer.put(nameMapping.policyId, policyNameMap);
 				}
-			}
-
-			return ret;
-		}
-
-		String getAccessType(Long accessTypeId) {
-			String ret = null;
-
-			if(accessTypeId != null) {
-				ret = accessTypes.get(accessTypeId);
 
-				if(ret == null) {
-					XXAccessTypeDef xAccessType = daoMgr.getXXAccessTypeDef().getById(accessTypeId);
-
-					if(xAccessType != null) {
-						ret = xAccessType.getName();
-
-						accessTypes.put(accessTypeId,  ret);
-					}
-				}
+				policyNameMap.put(nameMapping.oldName, nameMapping.currentName);
 			}
-
-			return ret;
 		}
 
-		String getConditionType(Long conditionDefId) {
-			String ret = null;
-
-			if(conditionDefId != null) {
-				ret = conditions.get(conditionDefId);
+		String getMappedName(Map<Long, Map<String, String>> nameMappingContainer, Long policyId, String nameToMap) {
+			Map<String, String> policyNameMap = nameMappingContainer.get(policyId);
 
-				if(ret == null) {
-					XXPolicyConditionDef xPolicyConditionDef = daoMgr.getXXPolicyConditionDef().getById(conditionDefId);
-
-					if(xPolicyConditionDef != null) {
-						ret = xPolicyConditionDef.getName();
-
-						conditions.put(conditionDefId,  ret);
-					}
-				}
-			}
-
-			return ret;
+			return policyNameMap != null ? policyNameMap.get(nameToMap) : null;
 		}
 
-		String getResourceName(Long resourceDefId) {
-			String ret = null;
-
-			if(resourceDefId != null) {
-				ret = resourceDefs.get(resourceDefId);
-
-				if(ret == null) {
-					XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId);
-
-					if(xResourceDef != null) {
-						ret = xResourceDef.getName();
+		void setGroupNameMapping(List<PolicyTextNameMap> groupNameMapping) {
+			setNameMapping(groupMappingsPerPolicy, groupNameMapping);
+		}
 
-						resourceDefs.put(resourceDefId,  ret);
-					}
-				}
-			}
+		void setUserNameMapping(List<PolicyTextNameMap> userNameMapping) {
+			setNameMapping(userMappingsPerPolicy, userNameMapping);
+		}
 
-			return ret;
+		void setAccessNameMapping(List<PolicyTextNameMap> accessNameMapping) {
+			setNameMapping(accessMappingsPerPolicy, accessNameMapping);
 		}
 
-		String getDataMaskName(Long dataMaskDefId) {
-			String ret = null;
+		public void setResourceNameMapping(List<PolicyTextNameMap> resourceNameMapping) {
+			setNameMapping(resourceMappingsPerPolicy, resourceNameMapping);
+		}
 
-			if(dataMaskDefId != null) {
-				ret = dataMasks.get(dataMaskDefId);
+		public void setDataMaskNameMapping(List<PolicyTextNameMap> dataMaskMapping) {
+			setNameMapping(dataMaskMappingsPerPolicy, dataMaskMapping);
+		}
 
-				if(ret == null) {
-					XXDataMaskTypeDef xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getById(dataMaskDefId);
+		public void setConditionNameMapping(List<PolicyTextNameMap> conditionNameMapping) {
+			setNameMapping(conditionMappingsPerPolicy, conditionNameMapping);
+		}
 
-					if(xDataMaskDef != null) {
-						ret = xDataMaskDef.getName();
+	}
 
-						dataMasks.put(dataMaskDefId,  ret);
-					}
-				}
-			}
+	public static class PolicyTextNameMap {
+		final Long   policyId;
+		final String oldName;
+		final String currentName;
 
-			return ret;
+		public PolicyTextNameMap(Long policyId, String oldName, String currentName) {
+			this.policyId    = policyId;
+			this.oldName     = oldName;
+			this.currentName = currentName;
 		}
 	}
 
-	static List<XXPolicy> asList(XXPolicy policy) {
-		List<XXPolicy> ret = new ArrayList<XXPolicy>();
+    static List<XXPolicy> asList(XXPolicy policy) {
+        List<XXPolicy> ret = new ArrayList<>();
 
-		if(policy != null) {
-			ret.add(policy);
-		}
+        if (policy != null) {
+            ret.add(policy);
+        }
 
-		return ret;
-	}
+        return ret;
+    }
 
 	class RetrieverContext {
-		final XXService                           service;
-		final ListIterator<XXPolicy>              iterPolicy;
-		final ListIterator<XXPolicyResource>      iterResources;
-		final ListIterator<XXPolicyResourceMap>   iterResourceMaps;
-		final ListIterator<XXPolicyItem>          iterPolicyItems;
-		final ListIterator<XXPolicyItemUserPerm>  iterUserPerms;
-		final ListIterator<XXPolicyItemGroupPerm> iterGroupPerms;
-		final ListIterator<XXPolicyItemAccess>    iterAccesses;
-		final ListIterator<XXPolicyItemCondition> iterConditions;
-		final ListIterator<XXPolicyItemDataMaskInfo>  iterDataMaskInfos;
-		final ListIterator<XXPolicyItemRowFilterInfo> iterRowFilterInfos;
-        final ListIterator<XXPolicyLabelMap> iterPolicyLabels;
+		final XXService              service;
+		final ListIterator<XXPolicy> iterPolicy;
+		final ListIterator<XXPolicyLabelMap> iterPolicyLabels;
 
 		RetrieverContext(XXService xService) {
-			Long serviceId = xService == null ? null : xService.getId();
-
-			List<XXPolicy>              xPolicies     = daoMgr.getXXPolicy().findByServiceId(serviceId);
-			List<XXPolicyResource>      xResources    = daoMgr.getXXPolicyResource().findByServiceId(serviceId);
-			List<XXPolicyResourceMap>   xResourceMaps = daoMgr.getXXPolicyResourceMap().findByServiceId(serviceId);
-			List<XXPolicyItem>          xPolicyItems  = daoMgr.getXXPolicyItem().findByServiceId(serviceId);
-			List<XXPolicyItemUserPerm>  xUserPerms    = daoMgr.getXXPolicyItemUserPerm().findByServiceId(serviceId);
-			List<XXPolicyItemGroupPerm> xGroupPerms   = daoMgr.getXXPolicyItemGroupPerm().findByServiceId(serviceId);
-			List<XXPolicyItemAccess>    xAccesses     = daoMgr.getXXPolicyItemAccess().findByServiceId(serviceId);
-			List<XXPolicyItemCondition> xConditions   = daoMgr.getXXPolicyItemCondition().findByServiceId(serviceId);
-			List<XXPolicyItemDataMaskInfo>  xDataMaskInfos  = daoMgr.getXXPolicyItemDataMaskInfo().findByServiceId(serviceId);
-			List<XXPolicyItemRowFilterInfo> xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByServiceId(serviceId);
-			List<XXPolicyLabelMap> xxPolicyLabelMap = daoMgr.getXXPolicyLabelMap().findByServiceId(serviceId);
-
-			this.service          = xService;
-			this.iterPolicy       = xPolicies.listIterator();
-			this.iterResources    = xResources.listIterator();
-			this.iterResourceMaps = xResourceMaps.listIterator();
-			this.iterPolicyItems  = xPolicyItems.listIterator();
-			this.iterUserPerms    = xUserPerms.listIterator();
-			this.iterGroupPerms   = xGroupPerms.listIterator();
-			this.iterAccesses     = xAccesses.listIterator();
-			this.iterConditions   = xConditions.listIterator();
-			this.iterDataMaskInfos  = xDataMaskInfos.listIterator();
-			this.iterRowFilterInfos = xRowFilterInfos.listIterator();
-            this.iterPolicyLabels = xxPolicyLabelMap.listIterator();
-		}
-
-		RetrieverContext(XXPolicy xPolicy) {
-			this(xPolicy, getXXService(xPolicy.getService()));
+			if (xService != null) {
+				Long serviceId = xService.getId();
+
+				lookupCache.setGroupNameMapping(daoMgr.getXXPolicyRefGroup().findUpdatedGroupNamesByService(serviceId));
+				lookupCache.setUserNameMapping(daoMgr.getXXPolicyRefUser().findUpdatedUserNamesByService(serviceId));
+				lookupCache.setAccessNameMapping(daoMgr.getXXPolicyRefAccessType().findUpdatedAccessNamesByService(serviceId));
+				lookupCache.setResourceNameMapping(daoMgr.getXXPolicyRefResource().findUpdatedResourceNamesByService(serviceId));
+				lookupCache.setDataMaskNameMapping(daoMgr.getXXPolicyRefDataMaskType().findUpdatedDataMaskNamesByService(serviceId));
+				lookupCache.setConditionNameMapping(daoMgr.getXXPolicyRefCondition().findUpdatedConditionNamesByService(serviceId));
+
+				this.service    = xService;
+				this.iterPolicy = daoMgr.getXXPolicy().findByServiceId(serviceId).listIterator();
+				this.iterPolicyLabels = daoMgr.getXXPolicyLabelMap().findByServiceId(serviceId).listIterator();
+			} else {
+				this.service    = null;
+				this.iterPolicy = null;
+				this.iterPolicyLabels = null;
+			}
 		}
 
 		RetrieverContext(XXPolicy xPolicy, XXService xService) {
-			Long policyId = xPolicy == null ? null : xPolicy.getId();
-
-			List<XXPolicy>              xPolicies     = asList(xPolicy);
-			List<XXPolicyResource>      xResources    = daoMgr.getXXPolicyResource().findByPolicyId(policyId);
-			List<XXPolicyResourceMap>   xResourceMaps = daoMgr.getXXPolicyResourceMap().findByPolicyId(policyId);
-			List<XXPolicyItem>          xPolicyItems  = daoMgr.getXXPolicyItem().findByPolicyId(policyId);
-			List<XXPolicyItemUserPerm>  xUserPerms    = daoMgr.getXXPolicyItemUserPerm().findByPolicyId(policyId);
-			List<XXPolicyItemGroupPerm> xGroupPerms   = daoMgr.getXXPolicyItemGroupPerm().findByPolicyId(policyId);
-			List<XXPolicyItemAccess>    xAccesses     = daoMgr.getXXPolicyItemAccess().findByPolicyId(policyId);
-			List<XXPolicyItemCondition> xConditions   = daoMgr.getXXPolicyItemCondition().findByPolicyId(policyId);
-			List<XXPolicyItemDataMaskInfo>  xDataMaskInfos  = daoMgr.getXXPolicyItemDataMaskInfo().findByPolicyId(policyId);
-			List<XXPolicyItemRowFilterInfo> xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByPolicyId(policyId);
-			List<XXPolicyLabelMap> xPolicyLabelMap = daoMgr.getXXPolicyLabelMap().findByPolicyId(policyId);
-
-			this.service          = xService;
-			this.iterPolicy       = xPolicies.listIterator();
-			this.iterResources    = xResources.listIterator();
-			this.iterResourceMaps = xResourceMaps.listIterator();
-			this.iterPolicyItems  = xPolicyItems.listIterator();
-			this.iterUserPerms    = xUserPerms.listIterator();
-			this.iterGroupPerms   = xGroupPerms.listIterator();
-			this.iterAccesses     = xAccesses.listIterator();
-			this.iterConditions   = xConditions.listIterator();
-			this.iterDataMaskInfos  = xDataMaskInfos.listIterator();
-			this.iterRowFilterInfos = xRowFilterInfos.listIterator();
-			this.iterPolicyLabels = xPolicyLabelMap.listIterator();
+			Long policyId = xPolicy.getId();
+
+			lookupCache.setGroupNameMapping(daoMgr.getXXPolicyRefGroup().findUpdatedGroupNamesByPolicy(policyId));
+			lookupCache.setUserNameMapping(daoMgr.getXXPolicyRefUser().findUpdatedUserNamesByPolicy(policyId));
+			lookupCache.setAccessNameMapping(daoMgr.getXXPolicyRefAccessType().findUpdatedAccessNamesByPolicy(policyId));
+			lookupCache.setResourceNameMapping(daoMgr.getXXPolicyRefResource().findUpdatedResourceNamesByPolicy(policyId));
+			lookupCache.setDataMaskNameMapping(daoMgr.getXXPolicyRefDataMaskType().findUpdatedDataMaskNamesByPolicy(policyId));
+			lookupCache.setConditionNameMapping(daoMgr.getXXPolicyRefCondition().findUpdatedConditionNamesByPolicy(policyId));
+
+			this.service    = xService;
+			this.iterPolicy = asList(xPolicy).listIterator();
+			this.iterPolicyLabels = daoMgr.getXXPolicyLabelMap().findByPolicyId(policyId).listIterator();
 		}
 
 		RangerPolicy getNextPolicy() {
 			RangerPolicy ret = null;
 
-			if(iterPolicy.hasNext()) {
+			if (service != null && iterPolicy != null && iterPolicy.hasNext()) {
 				XXPolicy xPolicy = iterPolicy.next();
 
-				if(xPolicy != null) {
-					ret = new RangerPolicy();
-
-					ret.setId(xPolicy.getId());
-					ret.setGuid(xPolicy.getGuid());
-					ret.setIsEnabled(xPolicy.getIsEnabled());
-					ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId()));
-					ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId()));
-					ret.setCreateTime(xPolicy.getCreateTime());
-					ret.setUpdateTime(xPolicy.getUpdateTime());
-					ret.setVersion(xPolicy.getVersion());
-					ret.setService(service == null ? null : service.getName());
-					ret.setName(StringUtils.trim(xPolicy.getName()));
-					ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType());
-					ret.setPolicyPriority(xPolicy.getPolicyPriority() == null ? RangerPolicy.POLICY_PRIORITY_NORMAL : xPolicy.getPolicyPriority());
-					ret.setDescription(xPolicy.getDescription());
-					ret.setResourceSignature(xPolicy.getResourceSignature());
-					ret.setIsAuditEnabled(xPolicy.getIsAuditEnabled());
-
-					Map<String, String> mapOfOptions = JsonUtils.jsonToMapStringString(xPolicy.getOptions());
-
-					if (MapUtils.isNotEmpty(mapOfOptions)) {
-						String validitySchedulesStr = mapOfOptions.get(RangerPolicyService.OPTION_POLICY_VALIDITY_SCHEDULES);
-
-						if (StringUtils.isNotEmpty(validitySchedulesStr)) {
-							List<RangerValiditySchedule> validitySchedules = JsonUtils.jsonToRangerValiditySchedule(validitySchedulesStr);
-
-							ret.setValiditySchedules(validitySchedules);
-						}
+				if (xPolicy != null) {
+					String policyText = xPolicy.getPolicyText();
+
+					ret = JsonUtils.jsonToObject(policyText, RangerPolicy.class);
+
+					if (ret != null) {
+						ret.setId(xPolicy.getId());
+						ret.setGuid(xPolicy.getGuid());
+						ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId()));
+						ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId()));
+						ret.setCreateTime(xPolicy.getCreateTime());
+						ret.setUpdateTime(xPolicy.getUpdateTime());
+						ret.setVersion(xPolicy.getVersion());
+						ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType());
+						ret.setService(service.getName());
+						updatePolicyReferenceFields(ret);
+						getPolicyLabels(ret);
 					}
-
-					getPolicyLabels(ret);
-					getResource(ret);
-					getPolicyItems(ret);
 				}
 			}
 
@@ -638,251 +524,121 @@ public class RangerPolicyRetriever {
 			}
 		}
 
-		List<RangerPolicy> getAllPolicies() {
-			List<RangerPolicy> ret = new ArrayList<RangerPolicy>();
-
-			while(iterPolicy.hasNext()) {
-				RangerPolicy policy = getNextPolicy();
-
-				if(policy != null) {
-					ret.add(policy);
-				}
-			}
-
-			if(! hasProcessedAll()) {
-				LOG.warn("getAllPolicies(): perhaps one or more policies got updated during retrieval. Falling back to secondary method");
-
-				ret = getAllPoliciesBySecondary();
-			}
-
-			return ret;
-		}
-
-		List<RangerPolicy> getAllPoliciesBySecondary() {
-			List<RangerPolicy> ret = null;
+		void updatePolicyReferenceFields(final RangerPolicy policy) {
+			final Long policyId = policy.getId();
 
-			if(service != null) {
-				List<XXPolicy> xPolicies = daoMgr.getXXPolicy().findByServiceId(service.getId());
+			Map<String, String> policyResourceNameMap = lookupCache.resourceMappingsPerPolicy.get(policyId);
 
-				if(CollectionUtils.isNotEmpty(xPolicies)) {
-					ret = new ArrayList<RangerPolicy>(xPolicies.size());
+			if (MapUtils.isNotEmpty(policyResourceNameMap) && CollectionUtils.containsAny(policyResourceNameMap.keySet(), policy.getResources().keySet())) {
+				Map<String, RangerPolicyResource> updatedResources = new HashMap<>();
 
-					for(XXPolicy xPolicy : xPolicies) {
-						RetrieverContext ctx = new RetrieverContext(xPolicy, service);
+				for (Map.Entry<String, RangerPolicyResource> entry : policy.getResources().entrySet()) {
+					String               resourceName   = entry.getKey();
+					RangerPolicyResource policyResource = entry.getValue();
+					String               updatedName    = policyResourceNameMap.get(resourceName);
 
-						RangerPolicy policy = ctx.getNextPolicy();
-
-						if(policy != null) {
-							ret.add(policy);
-						}
+					if (updatedName == null) {
+						updatedName = resourceName;
 					}
-				}
-			}
 
-			return ret;
-		}
-
-		private boolean hasProcessedAll() {
-			boolean moreToProcess =    iterPolicy.hasNext()
-									|| iterResources.hasNext()
-									|| iterResourceMaps.hasNext()
-									|| iterPolicyItems.hasNext()
-									|| iterUserPerms.hasNext()
-									|| iterGroupPerms.hasNext()
-									|| iterAccesses.hasNext()
-									|| iterConditions.hasNext()
-									|| iterDataMaskInfos.hasNext()
-									|| iterRowFilterInfos.hasNext()
-									|| iterPolicyLabels.hasNext();
-
-			return !moreToProcess;
-		}
-
-		private void getResource(RangerPolicy policy) {
-			while(iterResources.hasNext()) {
-				XXPolicyResource xResource = iterResources.next();
+					updatedResources.put(updatedName, policyResource);
+				}
 
-				if(xResource.getPolicyid().equals(policy.getId())) {
-					RangerPolicyResource resource = new RangerPolicyResource();
+				policy.setResources(updatedResources);
+			}
 
-					resource.setIsExcludes(xResource.getIsexcludes());
-					resource.setIsRecursive(xResource.getIsrecursive());
+			for (List<? extends RangerPolicyItem> policyItems :  PolicyRefUpdater.getAllPolicyItems(policy)) {
+				if (CollectionUtils.isEmpty(policyItems)) {
+					continue;
+				}
 
-					while(iterResourceMaps.hasNext()) {
-						XXPolicyResourceMap xResourceMap = iterResourceMaps.next();
+				for (RangerPolicyItem policyItem : policyItems) {
+					if (lookupCache.groupMappingsPerPolicy.containsKey(policyId)) {
+						List<String> updatedGroups = getUpdatedNames(lookupCache.groupMappingsPerPolicy, policyId, policyItem.getGroups());
 
-						if(xResourceMap.getResourceid().equals(xResource.getId())) {
-							resource.getValues().add(xResourceMap.getValue());
-						} else {
-							if(iterResourceMaps.hasPrevious()) {
-								iterResourceMaps.previous();
-							}
-							break;
+						if (updatedGroups != null) {
+							policyItem.setGroups(updatedGroups);
 						}
 					}
 
-					policy.getResources().put(lookupCache.getResourceName(xResource.getResdefid()), resource);
-				} else if(xResource.getPolicyid().compareTo(policy.getId()) > 0) {
-					if(iterResources.hasPrevious()) {
-						iterResources.previous();
-					}
-					break;
-				}
-			}
-		}
-
-		private void getPolicyItems(RangerPolicy policy) {
-			while(iterPolicyItems.hasNext()) {
-				XXPolicyItem xPolicyItem = iterPolicyItems.next();
+					if (lookupCache.userMappingsPerPolicy.containsKey(policyId)) {
+						List<String> updatedUsers = getUpdatedNames(lookupCache.userMappingsPerPolicy, policyId, policyItem.getUsers());
 
-				if(xPolicyItem.getPolicyid().equals(policy.getId())) {
-					final RangerPolicyItem          policyItem;
-					final RangerDataMaskPolicyItem  dataMaskPolicyItem;
-					final RangerRowFilterPolicyItem rowFilterPolicyItem;
-
-					if(xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) {
-						dataMaskPolicyItem  = new RangerDataMaskPolicyItem();
-						rowFilterPolicyItem = null;
-						policyItem          = dataMaskPolicyItem;
-					} else if(xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) {
-						dataMaskPolicyItem  = null;
-						rowFilterPolicyItem = new RangerRowFilterPolicyItem();
-						policyItem          = rowFilterPolicyItem;
-					} else {
-						dataMaskPolicyItem  = null;
-						rowFilterPolicyItem = null;
-						policyItem          = new RangerPolicyItem();
+						if (updatedUsers != null) {
+							policyItem.setUsers(updatedUsers);
+						}
 					}
 
+					if (lookupCache.accessMappingsPerPolicy.containsKey(policyId)) {
+						for (RangerPolicyItemAccess itemAccess : policyItem.getAccesses()) {
+							String updatedName = lookupCache.getMappedName(lookupCache.accessMappingsPerPolicy, policyId, itemAccess.getType());
 
-					while(iterAccesses.hasNext()) {
-						XXPolicyItemAccess xAccess = iterAccesses.next();
-
-						if(xAccess.getPolicyitemid().equals(xPolicyItem.getId())) {
-							policyItem.getAccesses().add(new RangerPolicyItemAccess(lookupCache.getAccessType(xAccess.getType()), xAccess.getIsallowed()));
-						} else {
-							if(iterAccesses.hasPrevious()) {
-								iterAccesses.previous();
+							if (updatedName != null) {
+								itemAccess.setType(updatedName);
 							}
-							break;
 						}
 					}
 
-					while(iterUserPerms.hasNext()) {
-						XXPolicyItemUserPerm xUserPerm = iterUserPerms.next();
+					if (lookupCache.conditionMappingsPerPolicy.containsKey(policyId)) {
+						for (RangerPolicyItemCondition condition : policyItem.getConditions()) {
+							String updatedName = lookupCache.getMappedName(lookupCache.conditionMappingsPerPolicy, policyId, condition.getType());
 
-						if(xUserPerm.getPolicyitemid().equals(xPolicyItem.getId())) {
-							String userName = lookupCache.getUserName(xUserPerm.getUserid());
-							if (userName != null) {
-								policyItem.getUsers().add(userName);
-							}
-						} else {
-							if(iterUserPerms.hasPrevious()) {
-								iterUserPerms.previous();
+							if (updatedName != null) {
+								condition.setType(updatedName);
 							}
-							break;
 						}
 					}
 
-					while(iterGroupPerms.hasNext()) {
-						XXPolicyItemGroupPerm xGroupPerm = iterGroupPerms.next();
+					if (policyItem instanceof RangerDataMaskPolicyItem && lookupCache.dataMaskMappingsPerPolicy.containsKey(policyId)) {
+						RangerDataMaskPolicyItem dataMaskItem = (RangerDataMaskPolicyItem) policyItem;
+						String                   updatedName  = lookupCache.getMappedName(lookupCache.dataMaskMappingsPerPolicy, policyId, dataMaskItem.getDataMaskInfo().getDataMaskType());
 
-						if(xGroupPerm.getPolicyitemid().equals(xPolicyItem.getId())) {
-							String groupName = lookupCache.getGroupName(xGroupPerm.getGroupid());
-							if (groupName != null) {
-								policyItem.getGroups().add(groupName);
-							}
-						} else {
-							if(iterGroupPerms.hasPrevious()) {
-								iterGroupPerms.previous();
-							}
-							break;
+						if (updatedName != null) {
+							dataMaskItem.getDataMaskInfo().setDataMaskType(updatedName);
 						}
 					}
+				}
+			}
+		}
 
-					RangerPolicyItemCondition condition         = null;
-					Long                      prevConditionType = null;
-					while(iterConditions.hasNext()) {
-						XXPolicyItemCondition xCondition = iterConditions.next();
+		List<String> getUpdatedNames(final Map<Long, Map<String, String>> nameMappingContainer, final Long policyId, final List<String> namesToMap) {
+			List<String>        ret           = null;
+			Map<String, String> policyNameMap = nameMappingContainer.get(policyId);
 
-						if(xCondition.getPolicyitemid().equals(xPolicyItem.getId())) {
-							if(! xCondition.getType().equals(prevConditionType)) {
-								condition = new RangerPolicyItemCondition();
-								condition.setType(lookupCache.getConditionType(xCondition.getType()));
-								condition.getValues().add(xCondition.getValue());
+			if (MapUtils.isNotEmpty(policyNameMap) && CollectionUtils.containsAny(policyNameMap.keySet(), namesToMap)) {
+				ret = new ArrayList<>();
 
-								policyItem.getConditions().add(condition);
+				for (String nameToMap : namesToMap) {
+					String mappedName = policyNameMap.get(nameToMap);
 
-								prevConditionType = xCondition.getType();
-							} else {
-								condition.getValues().add(xCondition.getValue());
-							}
-						} else {
-							if(iterConditions.hasPrevious()) {
-								iterConditions.previous();
-							}
-							break;
-						}
+					if (mappedName != null) {
+						ret.add(mappedName);
+					} else {
+						ret.add(nameToMap);
 					}
+				}
 
-					policyItem.setDelegateAdmin(xPolicyItem.getDelegateAdmin());
-
-					if(dataMaskPolicyItem != null) {
-						while (iterDataMaskInfos.hasNext()) {
-							XXPolicyItemDataMaskInfo xDataMaskInfo = iterDataMaskInfos.next();
-
-							if (xDataMaskInfo.getPolicyItemId().equals(xPolicyItem.getId())) {
-								dataMaskPolicyItem.setDataMaskInfo(new RangerPolicyItemDataMaskInfo(lookupCache.getDataMaskName(xDataMaskInfo.getType()), xDataMaskInfo.getConditionExpr(), xDataMaskInfo.getValueExpr()));
-							} else {
-								if (iterDataMaskInfos.hasPrevious()) {
-									iterDataMaskInfos.previous();
-								}
-								break;
-							}
-						}
-					}
+			}
 
-					if(rowFilterPolicyItem != null) {
-						while (iterRowFilterInfos.hasNext()) {
-							XXPolicyItemRowFilterInfo xRowFilterInfo = iterRowFilterInfos.next();
+			return ret;
+		}
 
-							if (xRowFilterInfo.getPolicyItemId().equals(xPolicyItem.getId())) {
-								rowFilterPolicyItem.setRowFilterInfo(new RangerPolicyItemRowFilterInfo(xRowFilterInfo.getFilterExpr()));
-							} else {
-								if (iterRowFilterInfos.hasPrevious()) {
-									iterRowFilterInfos.previous();
-								}
-								break;
-							}
-						}
-					}
+		List<RangerPolicy> getAllPolicies() {
+			List<RangerPolicy> ret = new ArrayList<>();
 
+			if (iterPolicy != null) {
+				while (iterPolicy.hasNext()) {
+					RangerPolicy policy = getNextPolicy();
 
-					int itemType = xPolicyItem.getItemType() == null ? RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW : xPolicyItem.getItemType();
-
-					if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) {
-						policy.getPolicyItems().add(policyItem);
-					} else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) {
-						policy.getDenyPolicyItems().add(policyItem);
-					} else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) {
-						policy.getAllowExceptions().add(policyItem);
-					} else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) {
-						policy.getDenyExceptions().add(policyItem);
-					} else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) {
-						policy.getDataMaskPolicyItems().add(dataMaskPolicyItem);
-					} else if(itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) {
-						policy.getRowFilterPolicyItems().add(rowFilterPolicyItem);
-					} else { // unknown itemType
-						LOG.warn("RangerPolicyRetriever.getPolicy(policyId=" + policy.getId() + "): ignoring unknown policyItemType " + itemType);
-					}
-				} else if(xPolicyItem.getPolicyid().compareTo(policy.getId()) > 0) {
-					if(iterPolicyItems.hasPrevious()) {
-						iterPolicyItems.previous();
+					if (policy != null) {
+						ret.add(policy);
 					}
-					break;
 				}
 			}
+
+			return ret;
 		}
 	}
+
 }
 

http://git-wip-us.apache.org/repos/asf/ranger/blob/3af1f59d/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 7875bc2..394c05c 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -19,25 +19,26 @@
 
 package org.apache.ranger.biz;
 
+import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
-import org.apache.ranger.authorization.utils.JsonUtils;
 import org.apache.ranger.authorization.utils.StringUtil;
 import org.apache.ranger.db.RangerDaoManager;
 import org.apache.ranger.entity.*;
 import org.apache.ranger.plugin.model.*;
-import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
 import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.apache.ranger.service.RangerServiceResourceService;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.TransactionStatus;
 import org.springframework.transaction.support.TransactionCallback;
@@ -48,34 +49,33 @@ public class RangerTagDBRetriever {
 	static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerTagDBRetriever");
 	public static final String OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN = "ranger.filter.tags.for.service.plugin";
 
+	public static final Type subsumedDataType = new TypeToken<List<RangerTagDef.RangerTagAttributeDef>>() {}.getType();
+
+	public static final Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")
+			.create();
+
 	private final RangerDaoManager daoMgr;
-	private final XXService xService;
 	private final LookupCache lookupCache;
 
-	private final PlatformTransactionManager  txManager;
-	private final TransactionTemplate         txTemplate;
-
 	private List<RangerServiceResource> serviceResources;
 	private Map<Long, RangerTagDef> tagDefs;
-	private Map<Long, RangerTag> tags;
-	private List<RangerTagResourceMap> tagResourceMaps;
 
-	private boolean filterForServicePlugin;
+	RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) {
 
-	public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) {
 		this.daoMgr = daoMgr;
-		this.txManager = txManager;
-		if (this.txManager != null) {
-			this.txTemplate = new TransactionTemplate(this.txManager);
-			this.txTemplate.setReadOnly(true);
+
+		final TransactionTemplate txTemplate;
+
+		if (txManager != null) {
+			txTemplate = new TransactionTemplate(txManager);
+			txTemplate.setReadOnly(true);
 		} else {
-			this.txTemplate = null;
+			txTemplate = null;
 		}
-		this.xService = xService;
 		this.lookupCache = new LookupCache();
 
 
-		if (this.daoMgr != null && this.xService != null) {
+		if (this.daoMgr != null && xService != null) {
 
 			RangerPerfTracer perf = null;
 
@@ -83,13 +83,11 @@ public class RangerTagDBRetriever {
 				perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerTagDBReceiver.getTags(serviceName=" + xService.getName());
 			}
 
-			filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false);
-
-			if (this.txTemplate == null) {
+			if (txTemplate == null) {
 				if (LOG.isDebugEnabled()) {
 					LOG.debug("Load Tags in the same thread and using an existing transaction");
 				}
-				if (initializeTagCache(xService) == false) {
+				if (!initializeTagCache(xService)) {
 					LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction");
 				}
 			} else {
@@ -112,20 +110,48 @@ public class RangerTagDBRetriever {
 		}
 	}
 
-	public List<RangerTagResourceMap> getTagResourceMaps() {
-		return tagResourceMaps;
-	}
-
-	public List<RangerServiceResource> getServiceResources() {
+	List<RangerServiceResource> getServiceResources() {
 		return serviceResources;
 	}
 
-	public Map<Long, RangerTagDef> getTagDefs() {
+	Map<Long, RangerTagDef> getTagDefs() {
 		return tagDefs;
 	}
 
-	public Map<Long, RangerTag> getTags() {
-		return tags;
+	Map<Long, RangerTag> getTags() {
+
+		Map<Long, RangerTag> ret = new HashMap<>();
+
+		if (CollectionUtils.isNotEmpty(serviceResources)) {
+			for (RangerServiceResource serviceResource : serviceResources) {
+				List<RangerTag> tags = lookupCache.serviceResourceToTags.get(serviceResource.getId());
+				if (CollectionUtils.isNotEmpty(tags)) {
+					for (RangerTag tag : tags) {
+						ret.put(tag.getId(), tag);
+					}
+				}
+			}
+		}
+
+		return ret;
+	}
+
+	Map<Long, List<Long>> getResourceToTagIds() {
+		Map<Long, List<Long>> ret = new HashMap<>();
+
+		if (CollectionUtils.isNotEmpty(serviceResources)) {
+			for (RangerServiceResource serviceResource : serviceResources) {
+				List<RangerTag> tags = lookupCache.serviceResourceToTags.get(serviceResource.getId());
+				if (CollectionUtils.isNotEmpty(tags)) {
+					List<Long> tagIds = new ArrayList<>();
+					ret.put(serviceResource.getId(), tagIds);
+					for (RangerTag tag : tags) {
+						tagIds.add(tag.getId());
+					}
+				}
+			}
+		}
+		return ret;
 	}
 
 	private boolean initializeTagCache(XXService xService) {
@@ -133,69 +159,23 @@ public class RangerTagDBRetriever {
 		try {
 			TagRetrieverServiceResourceContext  serviceResourceContext  = new TagRetrieverServiceResourceContext(xService);
 			TagRetrieverTagDefContext           tagDefContext           = new TagRetrieverTagDefContext(xService);
-			TagRetrieverTagContext              tagContext              = new TagRetrieverTagContext(xService);
 
 			serviceResources    = serviceResourceContext.getAllServiceResources();
 			tagDefs             = tagDefContext.getAllTagDefs();
-			tags                = tagContext.getAllTags();
-
-			tagResourceMaps     = getAllTagResourceMaps();
 
 			ret = true;
 		} catch (Exception ex) {
-			LOG.error("Failed to get tags for service:[" + xService.getName() + "]");
+			LOG.error("Failed to get tags for service:[" + xService.getName() + "]", ex);
 			serviceResources    = null;
 			tagDefs             = null;
-			tags                = null;
-			tagResourceMaps     = null;
 			ret = false;
 		}
 		return ret;
 	}
-	private List<RangerTagResourceMap> getAllTagResourceMaps() {
-
-		List<XXTagResourceMap> xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId());
-
-		ListIterator<XXTagResourceMap> iterTagResourceMap = xTagResourceMaps.listIterator();
-
-		List<RangerTagResourceMap> ret = new ArrayList<RangerTagResourceMap>();
-
-		while (iterTagResourceMap.hasNext()) {
-
-			XXTagResourceMap xTagResourceMap = iterTagResourceMap.next();
-
-			if (xTagResourceMap != null) {
-
-				RangerTagResourceMap tagResourceMap = new RangerTagResourceMap();
-
-				tagResourceMap.setId(xTagResourceMap.getId());
-				tagResourceMap.setGuid(xTagResourceMap.getGuid());
-				tagResourceMap.setCreatedBy(lookupCache.getUserScreenName(xTagResourceMap.getAddedByUserId()));
-				tagResourceMap.setUpdatedBy(lookupCache.getUserScreenName(xTagResourceMap.getUpdatedByUserId()));
-				tagResourceMap.setCreateTime(xTagResourceMap.getCreateTime());
-				tagResourceMap.setUpdateTime(xTagResourceMap.getUpdateTime());
-				tagResourceMap.setResourceId(xTagResourceMap.getResourceId());
-				tagResourceMap.setTagId(xTagResourceMap.getTagId());
-
-				ret.add(tagResourceMap);
-			}
-		}
-		return ret;
-	}
-
-	static <T> List<T> asList(T obj) {
-		List<T> ret = new ArrayList<T>();
-
-		if (obj != null) {
-			ret.add(obj);
-		}
-
-		return ret;
-	}
 
 	private class LookupCache {
-		final Map<Long, String> userScreenNames = new HashMap<Long, String>();
-		final Map<Long, String> resourceDefs = new HashMap<Long, String>();
+		final Map<Long, String> userScreenNames = new HashMap<>();
+		final Map<Long, List<RangerTag>> serviceResourceToTags = new HashMap<>();
 
 		String getUserScreenName(Long userId) {
 			String ret = null;
@@ -231,25 +211,6 @@ public class RangerTagDBRetriever {
 			return ret;
 		}
 
-		String getResourceName(Long resourceDefId) {
-			String ret = null;
-
-			if (resourceDefId != null) {
-				ret = resourceDefs.get(resourceDefId);
-
-				if (ret == null) {
-					XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId);
-
-					if (xResourceDef != null) {
-						ret = xResourceDef.getName();
-
-						resourceDefs.put(resourceDefId, ret);
-					}
-				}
-			}
-
-			return ret;
-		}
 	}
 
 	private class TagLoaderThread extends Thread {
@@ -289,39 +250,19 @@ public class RangerTagDBRetriever {
 
 		final XXService service;
 		final ListIterator<XXServiceResource> iterServiceResource;
-		final ListIterator<XXServiceResourceElement> iterServiceResourceElement;
-		final ListIterator<XXServiceResourceElementValue> iterServiceResourceElementValue;
 
 		TagRetrieverServiceResourceContext(XXService xService) {
 			Long serviceId = xService == null ? null : xService.getId();
-
-			List<XXServiceResource> xServiceResources = filterForServicePlugin ? daoMgr.getXXServiceResource().findForServicePlugin(serviceId) : daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(serviceId);
-			List<XXServiceResourceElement> xServiceResourceElements = filterForServicePlugin ? daoMgr.getXXServiceResourceElement().findForServicePlugin(serviceId) : daoMgr.getXXServiceResourceElement().findTaggedResourcesInServiceId(serviceId);
-			List<XXServiceResourceElementValue> xServiceResourceElementValues = filterForServicePlugin ? daoMgr.getXXServiceResourceElementValue().findForServicePlugin(serviceId) : daoMgr.getXXServiceResourceElementValue().findTaggedResourcesInServiceId(serviceId);
-
 			this.service = xService;
-			this.iterServiceResource = xServiceResources.listIterator();
-			this.iterServiceResourceElement = xServiceResourceElements.listIterator();
-			this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator();
-
-		}
 
-		TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) {
-			Long resourceId = xServiceResource == null ? null : xServiceResource.getId();
+			List<XXServiceResource> xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(serviceId);
 
-			List<XXServiceResource> xServiceResources = asList(xServiceResource);
-			List<XXServiceResourceElement> xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId);
-			List<XXServiceResourceElementValue> xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId);
-
-			this.service = xService;
 			this.iterServiceResource = xServiceResources.listIterator();
-			this.iterServiceResourceElement = xServiceResourceElements.listIterator();
-			this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator();
 
 		}
 
 		List<RangerServiceResource> getAllServiceResources() {
-			List<RangerServiceResource> ret = new ArrayList<RangerServiceResource>();
+			List<RangerServiceResource> ret = new ArrayList<>();
 
 			while (iterServiceResource.hasNext()) {
 				RangerServiceResource serviceResource = getNextServiceResource();
@@ -331,12 +272,6 @@ public class RangerTagDBRetriever {
 				}
 			}
 
-			if (!hasProcessedAll()) {
-				LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... ");
-
-				ret = getServiceResourcesBySecondary();
-			}
-
 			return ret;
 		}
 
@@ -346,7 +281,7 @@ public class RangerTagDBRetriever {
 			if (iterServiceResource.hasNext()) {
 				XXServiceResource xServiceResource = iterServiceResource.next();
 
-				if (xServiceResource != null) {
+				if (xServiceResource != null && StringUtils.isNotEmpty(xServiceResource.getTags())) {
 					ret = new RangerServiceResource();
 
 					ret.setId(xServiceResource.getId());
@@ -359,108 +294,35 @@ public class RangerTagDBRetriever {
 					ret.setVersion(xServiceResource.getVersion());
 					ret.setResourceSignature(xServiceResource.getResourceSignature());
 
-					getServiceResourceElements(ret);
-				}
-			}
-
-			return ret;
-		}
-
-		void getServiceResourceElements(RangerServiceResource serviceResource) {
-			while (iterServiceResourceElement.hasNext()) {
-				XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next();
-
-				if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) {
-					RangerPolicyResource resource = new RangerPolicyResource();
+					Map<String, RangerPolicy.RangerPolicyResource> serviceResourceElements = gsonBuilder.fromJson(xServiceResource.getServiceResourceElements(), RangerServiceResourceService.subsumedDataType);
+					ret.setResourceElements(serviceResourceElements);
 
-					resource.setIsExcludes(xServiceResourceElement.getIsExcludes());
-					resource.setIsRecursive(xServiceResourceElement.getIsRecursive());
-
-					while (iterServiceResourceElementValue.hasNext()) {
-						XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next();
-
-						if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) {
-							resource.getValues().add(xServiceResourceElementValue.getValue());
-						} else {
-							if (iterServiceResourceElementValue.hasPrevious()) {
-								iterServiceResourceElementValue.previous();
-							}
-							break;
-						}
-					}
-
-					serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource);
-				} else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) {
-					if (iterServiceResourceElement.hasPrevious()) {
-						iterServiceResourceElement.previous();
-					}
-					break;
+					List<RangerTag> tags = gsonBuilder.fromJson(xServiceResource.getTags(), RangerServiceResourceService.duplicatedDataType);
+					lookupCache.serviceResourceToTags.put(xServiceResource.getId(), tags);
 				}
 			}
-		}
-
-		boolean hasProcessedAll() {
-			boolean moreToProcess = iterServiceResource.hasNext()
-					|| iterServiceResourceElement.hasNext()
-					|| iterServiceResourceElementValue.hasNext();
-			return !moreToProcess;
-		}
-
-		List<RangerServiceResource> getServiceResourcesBySecondary() {
-			List<RangerServiceResource> ret = null;
-
-			if (service != null) {
-				List<XXServiceResource> xServiceResources = filterForServicePlugin ? daoMgr.getXXServiceResource().findForServicePlugin(service.getId()) : daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(service.getId());
 
-				if (CollectionUtils.isNotEmpty(xServiceResources)) {
-					ret = new ArrayList<RangerServiceResource>(xServiceResources.size());
-
-					for (XXServiceResource xServiceResource : xServiceResources) {
-						TagRetrieverServiceResourceContext ctx = new TagRetrieverServiceResourceContext(xServiceResource, service);
-
-						RangerServiceResource serviceResource = ctx.getNextServiceResource();
-
-						if (serviceResource != null) {
-							ret.add(serviceResource);
-						}
-					}
-				}
-			}
 			return ret;
 		}
+
 	}
 
 	private class TagRetrieverTagDefContext {
 
 		final XXService service;
 		final ListIterator<XXTagDef> iterTagDef;
-		final ListIterator<XXTagAttributeDef> iterTagAttributeDef;
-
 
 		TagRetrieverTagDefContext(XXService xService) {
 			Long serviceId = xService == null ? null : xService.getId();
 
-			List<XXTagDef> xTagDefs = filterForServicePlugin ? daoMgr.getXXTagDef().findForServicePlugin(serviceId) : daoMgr.getXXTagDef().findByServiceId(serviceId);
-			List<XXTagAttributeDef> xTagAttributeDefs = filterForServicePlugin ? daoMgr.getXXTagAttributeDef().findForServicePlugin(serviceId) : daoMgr.getXXTagAttributeDef().findByServiceId(serviceId);
-
-			this.service = xService;
-			this.iterTagDef = xTagDefs.listIterator();
-			this.iterTagAttributeDef = xTagAttributeDefs.listIterator();
-		}
-
-		TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) {
-			Long tagDefId = xTagDef == null ? null : xTagDef.getId();
-
-			List<XXTagDef> xTagDefs = asList(xTagDef);
-			List<XXTagAttributeDef> xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId);
+			List<XXTagDef> xTagDefs = daoMgr.getXXTagDef().findByServiceId(serviceId);
 
 			this.service = xService;
 			this.iterTagDef = xTagDefs.listIterator();
-			this.iterTagAttributeDef = xTagAttributeDefs.listIterator();
 		}
 
 		Map<Long, RangerTagDef> getAllTagDefs() {
-			Map<Long, RangerTagDef> ret = new HashMap<Long, RangerTagDef>();
+			Map<Long, RangerTagDef> ret = new HashMap<>();
 
 			while (iterTagDef.hasNext()) {
 				RangerTagDef tagDef = getNextTagDef();
@@ -470,13 +332,6 @@ public class RangerTagDBRetriever {
 				}
 			}
 
-			if (!hasProcessedAllTagDefs()) {
-				LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval.  Using fallback ... ");
-
-				ret = getTagDefsBySecondary();
-
-			}
-
 			return ret;
 		}
 
@@ -499,203 +354,14 @@ public class RangerTagDBRetriever {
 					ret.setVersion(xTagDef.getVersion());
 					ret.setName(xTagDef.getName());
 					ret.setSource(xTagDef.getSource());
-
-					getTagAttributeDefs(ret);
+					List<RangerTagDef.RangerTagAttributeDef> attributeDefs = gsonBuilder.fromJson(xTagDef.getTagAttrDefs(), RangerTagDBRetriever.subsumedDataType);
+					ret.setAttributeDefs(attributeDefs);
 				}
 			}
 
 			return ret;
 		}
 
-		void getTagAttributeDefs(RangerTagDef tagDef) {
-			while (iterTagAttributeDef.hasNext()) {
-				XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next();
-
-				if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) {
-					RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef();
-
-					tagAttributeDef.setName(xTagAttributeDef.getName());
-					tagAttributeDef.setType(xTagAttributeDef.getType());
-
-					tagDef.getAttributeDefs().add(tagAttributeDef);
-				} else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) {
-					if (iterTagAttributeDef.hasPrevious()) {
-						iterTagAttributeDef.previous();
-					}
-					break;
-				}
-			}
-		}
-
-		boolean hasProcessedAllTagDefs() {
-			boolean moreToProcess = iterTagAttributeDef.hasNext();
-			return !moreToProcess;
-		}
-
-		Map<Long, RangerTagDef> getTagDefsBySecondary() {
-			Map<Long, RangerTagDef> ret = null;
-
-			if (service != null) {
-				List<XXTagDef> xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId());
-
-				if (CollectionUtils.isNotEmpty(xTagDefs)) {
-					ret = new HashMap<Long, RangerTagDef>(xTagDefs.size());
-
-					for (XXTagDef xTagDef : xTagDefs) {
-						TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service);
-
-						RangerTagDef tagDef = ctx.getNextTagDef();
-
-						if (tagDef != null) {
-							ret.put(tagDef.getId(), tagDef);
-						}
-					}
-				}
-			}
-			return ret;
-		}
 	}
 
-	private class TagRetrieverTagContext {
-
-		final XXService service;
-		final ListIterator<XXTag> iterTag;
-		final ListIterator<XXTagAttribute> iterTagAttribute;
-
-		TagRetrieverTagContext(XXService xService) {
-			Long serviceId = xService == null ? null : xService.getId();
-
-			List<XXTag> xTags = filterForServicePlugin ? daoMgr.getXXTag().findForServicePlugin(serviceId) : daoMgr.getXXTag().findByServiceId(serviceId);
-			List<XXTagAttribute> xTagAttributes = filterForServicePlugin ? daoMgr.getXXTagAttribute().findForServicePlugin(serviceId) : daoMgr.getXXTagAttribute().findByServiceId(serviceId);
-
-			this.service = xService;
-			this.iterTag = xTags.listIterator();
-			this.iterTagAttribute = xTagAttributes.listIterator();
-
-		}
-
-		TagRetrieverTagContext(XXTag xTag, XXService xService) {
-			Long tagId = xTag == null ? null : xTag.getId();
-
-			List<XXTag> xTags = asList(xTag);
-			List<XXTagAttribute> xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId);
-
-			this.service = xService;
-			this.iterTag = xTags.listIterator();
-			this.iterTagAttribute = xTagAttributes.listIterator();
-		}
-
-
-		Map<Long, RangerTag> getAllTags() {
-			Map<Long, RangerTag> ret = new HashMap<Long, RangerTag>();
-
-			while (iterTag.hasNext()) {
-				RangerTag tag = getNextTag();
-
-				if (tag != null) {
-					ret.put(tag.getId(), tag);
-				}
-			}
-
-			if (!hasProcessedAllTags()) {
-				LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... ");
-
-				ret = getTagsBySecondary();
-			}
-
-			return ret;
-		}
-
-		RangerTag getNextTag() {
-			RangerTag ret = null;
-
-			if (iterTag.hasNext()) {
-				XXTag xTag = iterTag.next();
-
-				if (xTag != null) {
-					ret = new RangerTag();
-
-					ret.setId(xTag.getId());
-					ret.setGuid(xTag.getGuid());
-					ret.setOwner(xTag.getOwner());
-					ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId()));
-					ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId()));
-					ret.setCreateTime(xTag.getCreateTime());
-					ret.setUpdateTime(xTag.getUpdateTime());
-					ret.setVersion(xTag.getVersion());
-
-					Map<String, String> mapOfOptions = JsonUtils.jsonToMapStringString(xTag.getOptions());
-
-					if (MapUtils.isNotEmpty(mapOfOptions)) {
-						String validityPeriodsStr = mapOfOptions.get(RangerTag.OPTION_TAG_VALIDITY_PERIODS);
-
-						if (StringUtils.isNotEmpty(validityPeriodsStr)) {
-							List<RangerValiditySchedule> validityPeriods = JsonUtils.jsonToRangerValiditySchedule(validityPeriodsStr);
-
-							ret.setValidityPeriods(validityPeriods);
-						}
-					}
-
-					Map<Long, RangerTagDef> tagDefs = getTagDefs();
-					if (tagDefs != null) {
-						RangerTagDef tagDef = tagDefs.get(xTag.getType());
-						if (tagDef != null) {
-							ret.setType(tagDef.getName());
-						}
-					}
-
-					getTagAttributes(ret);
-				}
-			}
-
-			return ret;
-		}
-
-		void getTagAttributes(RangerTag tag) {
-			while (iterTagAttribute.hasNext()) {
-				XXTagAttribute xTagAttribute = iterTagAttribute.next();
-
-				if (xTagAttribute.getTagId().equals(tag.getId())) {
-					String attributeName = xTagAttribute.getName();
-					String attributeValue = xTagAttribute.getValue();
-
-					tag.getAttributes().put(attributeName, attributeValue);
-				} else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) {
-					if (iterTagAttribute.hasPrevious()) {
-						iterTagAttribute.previous();
-					}
-					break;
-				}
-			}
-		}
-
-		boolean hasProcessedAllTags() {
-			boolean moreToProcess = iterTagAttribute.hasNext();
-			return !moreToProcess;
-		}
-
-		Map<Long, RangerTag> getTagsBySecondary() {
-			Map<Long, RangerTag> ret = null;
-
-			if (service != null) {
-				List<XXTag> xTags = daoMgr.getXXTag().findByServiceId(service.getId());
-
-				if (CollectionUtils.isNotEmpty(xTags)) {
-					ret = new HashMap<Long, RangerTag>(xTags.size());
-
-					for (XXTag xTag : xTags) {
-						TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service);
-
-						RangerTag tag = ctx.getNextTag();
-
-						if (tag != null) {
-							ret.put(tag.getId(), tag);
-						}
-					}
-				}
-			}
-			return ret;
-		}
-	}
 }
-


Mime
View raw message