ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ab...@apache.org
Subject [3/9] ranger git commit: RANGER-2203, RANGER-2219: Review and update database schema for ranger policies and tag objects to minimize database queries/updates
Date Fri, 19 Oct 2018 17:01:34 GMT
http://git-wip-us.apache.org/repos/asf/ranger/blob/48cfb403/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java
new file mode 100644
index 0000000..6a21eff
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java
@@ -0,0 +1,1188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *	 http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.patch;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Logger;
+import org.apache.ranger.authorization.utils.JsonUtils;
+import org.apache.ranger.authorization.utils.StringUtil;
+import org.apache.ranger.biz.PolicyRefUpdater;
+import org.apache.ranger.biz.ServiceDBStore;
+import org.apache.ranger.db.RangerDaoManager;
+import org.apache.ranger.db.XXGroupDao;
+import org.apache.ranger.db.XXPolicyDao;
+import org.apache.ranger.db.XXPolicyRefAccessTypeDao;
+import org.apache.ranger.db.XXPolicyRefConditionDao;
+import org.apache.ranger.db.XXPolicyRefDataMaskTypeDao;
+import org.apache.ranger.db.XXPolicyRefGroupDao;
+import org.apache.ranger.db.XXPolicyRefResourceDao;
+import org.apache.ranger.db.XXPolicyRefUserDao;
+import org.apache.ranger.db.XXUserDao;
+import org.apache.ranger.entity.XXAccessTypeDef;
+import org.apache.ranger.entity.XXDataMaskTypeDef;
+import org.apache.ranger.entity.XXGroup;
+import org.apache.ranger.entity.XXPolicy;
+import org.apache.ranger.entity.XXPolicyConditionDef;
+import org.apache.ranger.entity.XXPolicyItem;
+import org.apache.ranger.entity.XXPolicyItemAccess;
+import org.apache.ranger.entity.XXPolicyItemCondition;
+import org.apache.ranger.entity.XXPolicyItemDataMaskInfo;
+import org.apache.ranger.entity.XXPolicyItemGroupPerm;
+import org.apache.ranger.entity.XXPolicyItemRowFilterInfo;
+import org.apache.ranger.entity.XXPolicyItemUserPerm;
+import org.apache.ranger.entity.XXPolicyLabel;
+import org.apache.ranger.entity.XXPolicyLabelMap;
+import org.apache.ranger.entity.XXPolicyRefAccessType;
+import org.apache.ranger.entity.XXPolicyRefCondition;
+import org.apache.ranger.entity.XXPolicyRefDataMaskType;
+import org.apache.ranger.entity.XXPolicyRefGroup;
+import org.apache.ranger.entity.XXPolicyRefResource;
+import org.apache.ranger.entity.XXPolicyRefUser;
+import org.apache.ranger.entity.XXPolicyResource;
+import org.apache.ranger.entity.XXPolicyResourceMap;
+import org.apache.ranger.entity.XXPortalUser;
+import org.apache.ranger.entity.XXResourceDef;
+import org.apache.ranger.entity.XXService;
+import org.apache.ranger.entity.XXServiceDef;
+import org.apache.ranger.entity.XXUser;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem;
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerValiditySchedule;
+import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator;
+import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.apache.ranger.plugin.util.SearchFilter;
+import org.apache.ranger.service.RangerPolicyService;
+import org.apache.ranger.util.CLIUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
+
+/**
+ * Consolidates Ranger policy details into a JSON string and stores it into a
+ * column in x_policy table After running this patch Ranger policy can be
+ * completely read/saved into x_policy table and some related Ref tables (which
+ * maintain ID->String mapping for each policy).
+ *
+ */
+@Component
+public class PatchForUpdatingPolicyJson_J10019 extends BaseLoader {
+	private static final Logger logger = Logger.getLogger(PatchForUpdatingPolicyJson_J10019.class);
+
+	@Autowired
+	RangerDaoManager daoMgr;
+
+	@Autowired
+	ServiceDBStore svcStore;
+
+	@Autowired
+	@Qualifier(value = "transactionManager")
+	PlatformTransactionManager txManager;
+
+	@Autowired
+	PolicyRefUpdater policyRefUpdater;
+
+	private final Map<String, Long>              groupIdMap         = new HashMap<>();
+	private final Map<String, Long>              userIdMap          = new HashMap<>();
+	private final Map<String, Map<String, Long>> resourceNameIdMap  = new HashMap<>();
+	private final Map<String, Map<String, Long>> accessTypeIdMap    = new HashMap<>();
+	private final Map<String, Map<String, Long>> conditionNameIdMap = new HashMap<>();
+	private final Map<String, Map<String, Long>> dataMaskTypeIdMap  = new HashMap<>();
+
+	public static void main(String[] args) {
+		logger.info("main()");
+		try {
+			PatchForUpdatingPolicyJson_J10019 loader = (PatchForUpdatingPolicyJson_J10019) CLIUtil.getBean(PatchForUpdatingPolicyJson_J10019.class);
+
+			loader.init();
+
+			while (loader.isMoreToProcess()) {
+				loader.load();
+			}
+
+			logger.info("Load complete. Exiting!!!");
+
+			System.exit(0);
+		} catch (Exception e) {
+			logger.error("Error loading", e);
+			System.exit(1);
+		}
+	}
+
+	@Override
+	public void init() throws Exception {
+		// Do Nothing
+	}
+
+	@Override
+	public void execLoad() {
+		logger.info("==> PatchForUpdatingPolicyJson.execLoad()");
+
+		try {
+			updateRangerPolicyTableWithPolicyJson();
+		} catch (Exception e) {
+			logger.error("Error while updateRangerPolicyTableWithPolicyJson()", e);
+			System.exit(1);
+		}
+
+		logger.info("<== PatchForUpdatingPolicyJson.execLoad()");
+	}
+
+	@Override
+	public void printStats() {
+		logger.info("updateRangerPolicyTableWithPolicyJson data ");
+	}
+
+	private void updateRangerPolicyTableWithPolicyJson() throws Exception {
+		logger.info("==> updateRangerPolicyTableWithPolicyJson() ");
+
+		List<RangerService> allServices = svcStore.getServices(new SearchFilter());
+
+		if (CollectionUtils.isNotEmpty(allServices)) {
+			for (RangerService service : allServices) {
+				XXService dbService = daoMgr.getXXService().getById(service.getId());
+
+				logger.info("==> Port Policies of service(name=" + dbService.getName() + ")");
+
+				RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager);
+
+				List<RangerPolicy> policies = policyRetriever.getServicePolicies(dbService);
+
+				if (CollectionUtils.isNotEmpty(policies)) {
+					for (RangerPolicy policy : policies) {
+						policyRefUpdater.cleanupRefTables(policy);
+						portPolicy(service.getType(), policy);
+					}
+				}
+			}
+		}
+
+		logger.info("<== updateRangerPolicyTableWithPolicyJson() ");
+	}
+
+	private void portPolicy(String serviceType, RangerPolicy policy) throws Exception {
+		logger.info("==> portPolicy(id=" + policy.getId() + ")");
+
+		String policyText = JsonUtils.objectToJson(policy);
+
+		if (StringUtils.isEmpty(policyText)) {
+			throw new Exception("Failed to convert policy to json string. Policy: [id=" +  policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]");
+		}
+
+		XXPolicyDao policyDao = daoMgr.getXXPolicy();
+		XXPolicy    dbBean    = policyDao.getById(policy.getId());
+
+		dbBean.setPolicyText(policyText);
+
+		policyDao.update(dbBean);
+
+		try {
+            		Set<String> accesses = new HashSet<>();
+            		Set<String> users = new HashSet<>();
+            		Set<String> groups = new HashSet<>();
+            		Set<String> conditions = new HashSet<>();
+            		Set<String> dataMasks = new HashSet<>();
+
+            		buildLists(policy.getPolicyItems(), accesses, conditions, users, groups);
+            		buildLists(policy.getDenyPolicyItems(), accesses, conditions, users, groups);
+            		buildLists(policy.getAllowExceptions(), accesses, conditions, users, groups);
+            		buildLists(policy.getDenyExceptions(), accesses, conditions, users, groups);
+            		buildLists(policy.getDataMaskPolicyItems(), accesses, conditions, users, groups);
+            		buildLists(policy.getRowFilterPolicyItems(), accesses, conditions, users, groups);
+
+            		buildList(policy.getDataMaskPolicyItems(), dataMasks);
+
+            		addResourceDefRef(serviceType, policy);
+            		addUserNameRef(policy.getId(), users);
+            		addGroupNameRef(policy.getId(), groups);
+            		addAccessDefRef(serviceType, policy.getId(), accesses);
+            		addPolicyConditionDefRef(serviceType, policy.getId(), conditions);
+            		addDataMaskDefRef(serviceType, policy.getId(), dataMasks);
+        	} catch (Exception e) {
+		    logger.error("portPoliry(id=" + policy.getId() +") failed!!");
+		    logger.error("Offending policy:" + policyText);
+		    throw e;
+        	}
+
+		logger.info("<== portPolicy(id=" + policy.getId() + ")");
+	}
+
+	private void addResourceDefRef(String serviceType, RangerPolicy policy) throws Exception {
+		logger.info("==> addResourceDefRef(id=" + policy.getId() + ")");
+
+		Map<String, Long> serviceDefResourceNameIDMap = resourceNameIdMap.get(serviceType);
+
+		if (serviceDefResourceNameIDMap == null) {
+			serviceDefResourceNameIDMap = new HashMap<>();
+
+			resourceNameIdMap.put(serviceType, serviceDefResourceNameIDMap);
+
+			XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType);
+
+			for (XXResourceDef resourceDef : daoMgr.getXXResourceDef().findByServiceDefId(dbServiceDef.getId())) {
+				serviceDefResourceNameIDMap.put(resourceDef.getName(), resourceDef.getId());
+			}
+		}
+
+		Map<String, RangerPolicyResource> policyResources = policy.getResources();
+
+		if (MapUtils.isNotEmpty(policyResources)) {
+			XXPolicyRefResourceDao policyRefResourceDao = daoMgr.getXXPolicyRefResource();
+			Set<String>            resourceNames        = policyResources.keySet();
+
+			for (String resourceName : resourceNames) {
+				Long resourceDefId = serviceDefResourceNameIDMap.get(resourceName);
+
+				if (resourceDefId == null) {
+					throw new Exception(resourceName + ": unknown resource in policy [id=" +  policy.getId() + "; name=" + policy.getName() + "; serviceType=" + serviceType + "]. Known resources: " + serviceDefResourceNameIDMap.keySet());
+				}
+
+				// insert policy-id, resourceDefId, resourceName into Ref table
+				XXPolicyRefResource policyRefResource = new XXPolicyRefResource();
+
+				policyRefResource.setPolicyId(policy.getId());
+				policyRefResource.setResourceDefId(resourceDefId);
+				policyRefResource.setResourceName(resourceName);
+
+				policyRefResourceDao.create(policyRefResource);
+			}
+		}
+
+		logger.info("<== addResourceDefRef(id=" + policy.getId() + ")");
+	}
+
+	private void addUserNameRef(Long policyId, Set<String> users) throws Exception {
+		logger.info("==> addUserNameRef(id=" + policyId + ")");
+
+		XXPolicyRefUserDao policyRefUserDao = daoMgr.getXXPolicyRefUser();
+		XXUserDao          userDao          = daoMgr.getXXUser();
+
+		// insert policy-id, userName into Ref table
+		for (String user : users) {
+			Long userId = userIdMap.get(user);
+
+			if (userId == null) {
+				XXUser userObject = userDao.findByUserName(user);
+
+				if (userObject == null) {
+					throw new Exception(user + ": unknown user in policy [id=" + policyId + "]");
+				}
+
+				userId = userObject.getId();
+
+				userIdMap.put(user, userId);
+			}
+
+			XXPolicyRefUser policyRefUser = new XXPolicyRefUser();
+
+			policyRefUser.setPolicyId(policyId);
+			policyRefUser.setUserName(user);
+			policyRefUser.setUserId(userId);
+
+			policyRefUserDao.create(policyRefUser);
+		}
+
+		logger.info("<== addUserNameRef(id=" + policyId + ")");
+	}
+
+	private void addGroupNameRef(Long policyId, Set<String> groups) throws Exception {
+		logger.info("==> addGroupNameRef(id=" + policyId + ")");
+
+		// insert policy-id, groupName into Ref table
+		XXPolicyRefGroupDao policyRefGroupDao = daoMgr.getXXPolicyRefGroup();
+		XXGroupDao          groupDao          = daoMgr.getXXGroup();
+
+		for (String group : groups) {
+			Long groupId = groupIdMap.get(group);
+
+			if (groupId == null) {
+				XXGroup groupObject = groupDao.findByGroupName(group);
+
+				if (groupObject == null) {
+					throw new Exception(group + ": unknown group in policy [id=" + policyId + "]");
+				}
+
+				groupId = groupObject.getId();
+
+				groupIdMap.put(group, groupId);
+			}
+
+			XXPolicyRefGroup policyRefGroup = new XXPolicyRefGroup();
+
+			policyRefGroup.setPolicyId(policyId);
+			policyRefGroup.setGroupName(group);
+			policyRefGroup.setGroupId(groupId);
+
+			policyRefGroupDao.create(policyRefGroup);
+		}
+
+		logger.info("<== addGroupNameRef(id=" + policyId + ")");
+
+	}
+
+	private void addAccessDefRef(String serviceType, Long policyId, Set<String> accesses) throws Exception {
+		logger.info("==> addAccessDefRef(id=" + policyId + ")");
+		// insert policy-id, accessName into Ref table
+
+		Map<String, Long> serviceDefAccessTypeIDMap = accessTypeIdMap.get(serviceType);
+
+		if (serviceDefAccessTypeIDMap == null) {
+			serviceDefAccessTypeIDMap = new HashMap<>();
+
+			accessTypeIdMap.put(serviceType, serviceDefAccessTypeIDMap);
+
+			XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType);
+
+			for (XXAccessTypeDef accessTypeDef : daoMgr.getXXAccessTypeDef().findByServiceDefId(dbServiceDef.getId())) {
+				serviceDefAccessTypeIDMap.put(accessTypeDef.getName(), accessTypeDef.getId());
+			}
+		}
+
+		XXPolicyRefAccessTypeDao policyRefAccessTypeDao = daoMgr.getXXPolicyRefAccessType();
+
+		for (String access : accesses) {
+			Long accessTypeDefId = serviceDefAccessTypeIDMap.get(access);
+
+			if (accessTypeDefId == null) {
+				throw new Exception(access + ": unknown accessType in policy [id=" +  policyId + "; serviceType=" + serviceType + "]. Known accessTypes: " + serviceDefAccessTypeIDMap.keySet());
+			}
+
+			XXPolicyRefAccessType policyRefAccessType = new XXPolicyRefAccessType();
+
+			policyRefAccessType.setPolicyId(policyId);
+			policyRefAccessType.setAccessTypeName(access);
+			policyRefAccessType.setAccessDefId(accessTypeDefId);
+
+			policyRefAccessTypeDao.create(policyRefAccessType);
+		}
+
+		logger.info("<== addAccessDefRef(id=" + policyId + ")");
+	}
+
+	private void addPolicyConditionDefRef(String serviceType, Long policyId, Set<String> conditions) throws Exception {
+		logger.info("==> addPolicyConditionDefRef(id=" + policyId + ")");
+		// insert policy-id, conditionName into Ref table
+
+		Map<String, Long> serviceDefConditionNameIDMap = conditionNameIdMap.get(serviceType);
+
+		if (serviceDefConditionNameIDMap == null) {
+			serviceDefConditionNameIDMap = new HashMap<>();
+
+			conditionNameIdMap.put(serviceType, serviceDefConditionNameIDMap);
+
+			XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType);
+
+			for (XXPolicyConditionDef conditionDef : daoMgr.getXXPolicyConditionDef().findByServiceDefId(dbServiceDef.getId())) {
+				serviceDefConditionNameIDMap.put(conditionDef.getName(), conditionDef.getId());
+			}
+		}
+
+		XXPolicyRefConditionDao policyRefConditionDao = daoMgr.getXXPolicyRefCondition();
+
+		for (String condition : conditions) {
+			Long conditionDefId = serviceDefConditionNameIDMap.get(condition);
+
+			if (conditionDefId == null) {
+				throw new Exception(condition + ": unknown condition in policy [id=" +  policyId + "; serviceType=" + serviceType + "]. Known conditions are: " + serviceDefConditionNameIDMap.keySet());
+			}
+
+			XXPolicyRefCondition policyRefCondition = new XXPolicyRefCondition();
+
+			policyRefCondition.setPolicyId(policyId);
+			policyRefCondition.setConditionName(condition);
+			policyRefCondition.setConditionDefId(conditionDefId);
+
+			policyRefConditionDao.create(policyRefCondition);
+		}
+
+		logger.info("<== addPolicyConditionDefRef(id=" + policyId + ")");
+	}
+
+	private void addDataMaskDefRef(String serviceType, Long policyId, Set<String> datamasks) throws Exception {
+		logger.info("==> addDataMaskDefRef(id=" + policyId + ")");
+
+		// insert policy-id, datamaskName into Ref table
+
+		Map<String, Long> serviceDefDataMaskTypeIDMap = dataMaskTypeIdMap.get(serviceType);
+
+		if (serviceDefDataMaskTypeIDMap == null) {
+			serviceDefDataMaskTypeIDMap = new HashMap<>();
+
+			dataMaskTypeIdMap.put(serviceType, serviceDefDataMaskTypeIDMap);
+
+			XXServiceDef dbServiceDef = daoMgr.getXXServiceDef().findByName(serviceType);
+
+			for (XXDataMaskTypeDef dataMaskTypeDef : daoMgr.getXXDataMaskTypeDef().findByServiceDefId(dbServiceDef.getId())) {
+				serviceDefDataMaskTypeIDMap.put(dataMaskTypeDef.getName(), dataMaskTypeDef.getId());
+			}
+		}
+
+		XXPolicyRefDataMaskTypeDao policyRefDataMaskTypeDao = daoMgr.getXXPolicyRefDataMaskType();
+
+		for (String datamask : datamasks) {
+			Long dataMaskTypeId = serviceDefDataMaskTypeIDMap.get(datamask);
+
+			if (dataMaskTypeId == null) {
+				throw new Exception(datamask + ": unknown dataMaskType in policy [id=" +  policyId + "; serviceType=" + serviceType + "]. Known dataMaskTypes " + serviceDefDataMaskTypeIDMap.keySet());
+			}
+
+			XXPolicyRefDataMaskType policyRefDataMaskType = new XXPolicyRefDataMaskType();
+
+			policyRefDataMaskType.setPolicyId(policyId);
+			policyRefDataMaskType.setDataMaskTypeName(datamask);
+			policyRefDataMaskType.setDataMaskDefId(dataMaskTypeId);
+
+			policyRefDataMaskTypeDao.create(policyRefDataMaskType);
+		}
+
+		logger.info("<== addDataMaskDefRef(id=" + policyId + ")");
+
+	}
+
+	private void buildLists(List<? extends RangerPolicyItem> policyItems, Set<String> accesses, Set<String> conditions, Set<String> users, Set<String> groups) {
+		for (RangerPolicyItem item : policyItems) {
+			for (RangerPolicyItemAccess policyAccess : item.getAccesses()) {
+				accesses.add(policyAccess.getType());
+			}
+
+			for (RangerPolicyItemCondition policyCondition : item.getConditions()) {
+				conditions.add(policyCondition.getType());
+			}
+
+			users.addAll(item.getUsers());
+			groups.addAll(item.getGroups());
+		}
+	}
+
+	private void buildList(List<RangerDataMaskPolicyItem> dataMaskPolicyItems, Set<String> dataMasks) {
+		for (RangerDataMaskPolicyItem datMaskPolicyItem : dataMaskPolicyItems) {
+			dataMasks.add(datMaskPolicyItem.getDataMaskInfo().getDataMaskType());
+		}
+	}
+
+	static private class RangerPolicyRetriever {
+		static final Log LOG      = LogFactory.getLog(RangerPolicyRetriever.class);
+		static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever");
+
+		private final RangerDaoManager daoMgr;
+		private final LookupCache      lookupCache = new LookupCache();
+
+		private final PlatformTransactionManager txManager;
+		private final TransactionTemplate        txTemplate;
+
+		RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) {
+			this.daoMgr    = daoMgr;
+			this.txManager = txManager;
+
+			if (this.txManager != null) {
+				this.txTemplate = new TransactionTemplate(this.txManager);
+
+				this.txTemplate.setReadOnly(true);
+			} else {
+				this.txTemplate = null;
+			}
+		}
+
+		private class PolicyLoaderThread extends Thread {
+			final TransactionTemplate txTemplate;
+			final XXService           xService;
+			List<RangerPolicy>        policies;
+
+			PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+				this.txTemplate = txTemplate;
+				this.xService   = xService;
+			}
+
+			public List<RangerPolicy> getPolicies() {
+				return policies;
+			}
+
+			@Override
+			public void run() {
+				txTemplate.setReadOnly(true);
+				policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() {
+					@Override
+					public List<RangerPolicy> doInTransaction(TransactionStatus status) {
+						RetrieverContext ctx = new RetrieverContext(xService);
+						return ctx.getAllPolicies();
+					}
+				});
+			}
+		}
+
+		public List<RangerPolicy> getServicePolicies(final XXService xService) throws InterruptedException {
+			String serviceName = xService == null ? null : xService.getName();
+			Long   serviceId   = xService == null ? null : xService.getId();
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("==> RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + ")");
+			}
+
+			List<RangerPolicy> ret  = null;
+			RangerPerfTracer   perf = null;
+
+			if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+				perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ",serviceId=" + serviceId + ")");
+			}
+
+			if (xService != null) {
+				if (txTemplate == null) {
+					if (LOG.isDebugEnabled()) {
+						LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction");
+					}
+
+					RetrieverContext ctx = new RetrieverContext(xService);
+
+					ret = ctx.getAllPolicies();
+				} else {
+					if (LOG.isDebugEnabled()) {
+						LOG.debug("Retrieving policies in a new, read-only transaction");
+					}
+
+					PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService);
+					t.start();
+					t.join();
+					ret = t.getPolicies();
+				}
+			} else {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter");
+				}
+			}
+
+			RangerPerfTracer.log(perf);
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("<== RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + "): policyCount=" + (ret == null ? 0 : ret.size()));
+			}
+
+			return ret;
+		}
+
+		class LookupCache {
+			final Map<Long, String> userNames       = new HashMap<Long, String>();
+			final Map<Long, String> userScreenNames = new HashMap<Long, String>();
+			final Map<Long, String> groupNames      = new HashMap<Long, String>();
+			final Map<Long, String> accessTypes     = new HashMap<Long, String>();
+			final Map<Long, String> conditions      = new HashMap<Long, String>();
+			final Map<Long, String> resourceDefs    = new HashMap<Long, String>();
+			final Map<Long, String> dataMasks       = new HashMap<Long, String>();
+            final Map<Long, String> policyLabels    = new HashMap<Long, String>();
+
+			String getUserName(Long userId) {
+				String ret = null;
+
+				if (userId != null) {
+					ret = userNames.get(userId);
+
+					if (ret == null) {
+						XXUser user = daoMgr.getXXUser().getById(userId);
+
+						if (user != null) {
+							ret = user.getName(); // Name is `loginId`
+
+							userNames.put(userId, ret);
+						}
+					}
+				}
+
+				return ret;
+			}
+
+            String getPolicyLabelName(Long policyLabelId) {
+                String ret = null;
+
+                if (policyLabelId != null) {
+                    ret = policyLabels.get(policyLabelId);
+
+                    if (ret == null) {
+                        XXPolicyLabel xxPolicyLabel = daoMgr.getXXPolicyLabels().getById(policyLabelId);
+
+                        if (xxPolicyLabel != null) {
+                            ret = xxPolicyLabel.getPolicyLabel();
+
+                            policyLabels.put(policyLabelId, ret);
+                        }
+                    }
+                }
+
+                return ret;
+            }
+
+			String getUserScreenName(Long userId) {
+				String ret = null;
+
+				if (userId != null) {
+					ret = userScreenNames.get(userId);
+
+					if (ret == null) {
+						XXPortalUser user = daoMgr.getXXPortalUser().getById(userId);
+
+						if (user != null) {
+							ret = user.getPublicScreenName();
+
+							if (StringUtil.isEmpty(ret)) {
+								ret = user.getFirstName();
+
+								if (StringUtil.isEmpty(ret)) {
+									ret = user.getLoginId();
+								} else {
+									if (!StringUtil.isEmpty(user.getLastName())) {
+										ret += (" " + user.getLastName());
+									}
+								}
+							}
+
+							if (ret != null) {
+								userScreenNames.put(userId, ret);
+							}
+						}
+					}
+				}
+
+				return ret;
+			}
+
+			String getGroupName(Long groupId) {
+				String ret = null;
+
+				if (groupId != null) {
+					ret = groupNames.get(groupId);
+
+					if (ret == null) {
+						XXGroup group = daoMgr.getXXGroup().getById(groupId);
+
+						if (group != null) {
+							ret = group.getName();
+
+							groupNames.put(groupId, ret);
+						}
+					}
+				}
+
+				return ret;
+			}
+
+			String getAccessType(Long accessTypeId) {
+				String ret = null;
+
+				if (accessTypeId != null) {
+					ret = accessTypes.get(accessTypeId);
+
+					if (ret == null) {
+						XXAccessTypeDef xAccessType = daoMgr.getXXAccessTypeDef().getById(accessTypeId);
+
+						if (xAccessType != null) {
+							ret = xAccessType.getName();
+
+							accessTypes.put(accessTypeId, ret);
+						} else {
+							LOG.warn("getAccessType(): Canot find name for accessTypeId " + accessTypeId + ". This will cause Ranger policy migration to fail. Please check if all service-defs are migrated correctly!");
+						}
+					}
+				}
+
+				return ret;
+			}
+
+			String getConditionType(Long conditionDefId) {
+				String ret = null;
+
+				if (conditionDefId != null) {
+					ret = conditions.get(conditionDefId);
+
+					if (ret == null) {
+						XXPolicyConditionDef xPolicyConditionDef = daoMgr.getXXPolicyConditionDef()
+								.getById(conditionDefId);
+
+						if (xPolicyConditionDef != null) {
+							ret = xPolicyConditionDef.getName();
+
+							conditions.put(conditionDefId, ret);
+						}
+					}
+				}
+
+				return ret;
+			}
+
+			String getResourceName(Long resourceDefId) {
+				String ret = null;
+
+				if (resourceDefId != null) {
+					ret = resourceDefs.get(resourceDefId);
+
+					if (ret == null) {
+						XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId);
+
+						if (xResourceDef != null) {
+							ret = xResourceDef.getName();
+
+							resourceDefs.put(resourceDefId, ret);
+						}
+					}
+				}
+
+				return ret;
+			}
+
+			String getDataMaskName(Long dataMaskDefId) {
+				String ret = null;
+
+				if (dataMaskDefId != null) {
+					ret = dataMasks.get(dataMaskDefId);
+
+					if (ret == null) {
+						XXDataMaskTypeDef xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getById(dataMaskDefId);
+
+						if (xDataMaskDef != null) {
+							ret = xDataMaskDef.getName();
+
+							dataMasks.put(dataMaskDefId, ret);
+						}
+					}
+				}
+
+				return ret;
+			}
+		}
+
+		static List<XXPolicy> asList(XXPolicy policy) {
+			List<XXPolicy> ret = new ArrayList<XXPolicy>();
+
+			if (policy != null) {
+				ret.add(policy);
+			}
+
+			return ret;
+		}
+
+		class RetrieverContext {
+			final XXService                               service;
+			final ListIterator<XXPolicy>                  iterPolicy;
+			final ListIterator<XXPolicyResource>          iterResources;
+			final ListIterator<XXPolicyResourceMap>       iterResourceMaps;
+			final ListIterator<XXPolicyItem>              iterPolicyItems;
+			final ListIterator<XXPolicyItemUserPerm>      iterUserPerms;
+			final ListIterator<XXPolicyItemGroupPerm>     iterGroupPerms;
+			final ListIterator<XXPolicyItemAccess>        iterAccesses;
+			final ListIterator<XXPolicyItemCondition>     iterConditions;
+			final ListIterator<XXPolicyItemDataMaskInfo>  iterDataMaskInfos;
+			final ListIterator<XXPolicyItemRowFilterInfo> iterRowFilterInfos;
+            final ListIterator<XXPolicyLabelMap>          iterPolicyLabels;
+
+			RetrieverContext(XXService xService) {
+				Long           serviceId = xService == null ? null : xService.getId();
+				List<XXPolicy> xPolicies = daoMgr.getXXPolicy().findByServiceId(serviceId);
+
+				this.service    = xService;
+				this.iterPolicy = xPolicies.listIterator();
+
+				List<XXPolicyResource>          xResources      = daoMgr.getXXPolicyResource().findByServiceId(serviceId);
+				List<XXPolicyResourceMap>       xResourceMaps   = daoMgr.getXXPolicyResourceMap().findByServiceId(serviceId);
+				List<XXPolicyItem>              xPolicyItems    = daoMgr.getXXPolicyItem().findByServiceId(serviceId);
+				List<XXPolicyItemUserPerm>      xUserPerms      = daoMgr.getXXPolicyItemUserPerm().findByServiceId(serviceId);
+				List<XXPolicyItemGroupPerm>     xGroupPerms     = daoMgr.getXXPolicyItemGroupPerm().findByServiceId(serviceId);
+				List<XXPolicyItemAccess>        xAccesses       = daoMgr.getXXPolicyItemAccess().findByServiceId(serviceId);
+				List<XXPolicyItemCondition>     xConditions     = daoMgr.getXXPolicyItemCondition().findByServiceId(serviceId);
+				List<XXPolicyItemDataMaskInfo>  xDataMaskInfos  = daoMgr.getXXPolicyItemDataMaskInfo().findByServiceId(serviceId);
+				List<XXPolicyItemRowFilterInfo> xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByServiceId(serviceId);
+                List<XXPolicyLabelMap>          xPolicyLabelMap = daoMgr.getXXPolicyLabelMap().findByServiceId(serviceId);
+
+				this.iterResources      = xResources.listIterator();
+				this.iterResourceMaps   = xResourceMaps.listIterator();
+				this.iterPolicyItems    = xPolicyItems.listIterator();
+				this.iterUserPerms      = xUserPerms.listIterator();
+				this.iterGroupPerms     = xGroupPerms.listIterator();
+				this.iterAccesses       = xAccesses.listIterator();
+				this.iterConditions     = xConditions.listIterator();
+				this.iterDataMaskInfos  = xDataMaskInfos.listIterator();
+				this.iterRowFilterInfos = xRowFilterInfos.listIterator();
+                this.iterPolicyLabels   = xPolicyLabelMap.listIterator();
+			}
+
+			RetrieverContext(XXPolicy xPolicy, XXService xService) {
+				Long           policyId  = xPolicy == null ? null : xPolicy.getId();
+				List<XXPolicy> xPolicies = asList(xPolicy);
+
+				this.service    = xService;
+				this.iterPolicy = xPolicies.listIterator();
+
+				List<XXPolicyResource>          xResources      = daoMgr.getXXPolicyResource().findByPolicyId(policyId);
+				List<XXPolicyResourceMap>       xResourceMaps   = daoMgr.getXXPolicyResourceMap().findByPolicyId(policyId);
+				List<XXPolicyItem>              xPolicyItems    = daoMgr.getXXPolicyItem().findByPolicyId(policyId);
+				List<XXPolicyItemUserPerm>      xUserPerms      = daoMgr.getXXPolicyItemUserPerm().findByPolicyId(policyId);
+				List<XXPolicyItemGroupPerm>     xGroupPerms     = daoMgr.getXXPolicyItemGroupPerm().findByPolicyId(policyId);
+				List<XXPolicyItemAccess>        xAccesses       = daoMgr.getXXPolicyItemAccess().findByPolicyId(policyId);
+				List<XXPolicyItemCondition>     xConditions     = daoMgr.getXXPolicyItemCondition().findByPolicyId(policyId);
+				List<XXPolicyItemDataMaskInfo>  xDataMaskInfos  = daoMgr.getXXPolicyItemDataMaskInfo().findByPolicyId(policyId);
+				List<XXPolicyItemRowFilterInfo> xRowFilterInfos = daoMgr.getXXPolicyItemRowFilterInfo().findByPolicyId(policyId);
+                List<XXPolicyLabelMap>          xPolicyLabelMap = daoMgr.getXXPolicyLabelMap().findByPolicyId(policyId);
+
+				this.iterResources      = xResources.listIterator();
+				this.iterResourceMaps   = xResourceMaps.listIterator();
+				this.iterPolicyItems    = xPolicyItems.listIterator();
+				this.iterUserPerms      = xUserPerms.listIterator();
+				this.iterGroupPerms     = xGroupPerms.listIterator();
+				this.iterAccesses       = xAccesses.listIterator();
+				this.iterConditions     = xConditions.listIterator();
+				this.iterDataMaskInfos  = xDataMaskInfos.listIterator();
+				this.iterRowFilterInfos = xRowFilterInfos.listIterator();
+                this.iterPolicyLabels   = xPolicyLabelMap.listIterator();
+			}
+
+			RangerPolicy getNextPolicy() {
+				RangerPolicy ret = null;
+
+				if (iterPolicy.hasNext()) {
+					XXPolicy xPolicy = iterPolicy.next();
+
+					if (xPolicy != null) {
+						ret = new RangerPolicy();
+
+						ret.setId(xPolicy.getId());
+						ret.setGuid(xPolicy.getGuid());
+						ret.setIsEnabled(xPolicy.getIsEnabled());
+						ret.setCreatedBy(lookupCache.getUserScreenName(xPolicy.getAddedByUserId()));
+						ret.setUpdatedBy(lookupCache.getUserScreenName(xPolicy.getUpdatedByUserId()));
+						ret.setCreateTime(xPolicy.getCreateTime());
+						ret.setUpdateTime(xPolicy.getUpdateTime());
+						ret.setVersion(xPolicy.getVersion());
+						ret.setService(service == null ? null : service.getName());
+						ret.setName(StringUtils.trim(xPolicy.getName()));
+						ret.setPolicyType(xPolicy.getPolicyType() == null ? RangerPolicy.POLICY_TYPE_ACCESS : xPolicy.getPolicyType());
+						ret.setDescription(xPolicy.getDescription());
+						ret.setResourceSignature(xPolicy.getResourceSignature());
+						ret.setIsAuditEnabled(xPolicy.getIsAuditEnabled());
+						ret.setPolicyPriority(xPolicy.getPolicyPriority());
+
+						Map<String, String> mapOfOptions = JsonUtils.jsonToMapStringString(xPolicy.getOptions());
+
+						if (MapUtils.isNotEmpty(mapOfOptions)) {
+							String validitySchedulesStr = mapOfOptions.get(RangerPolicyService.OPTION_POLICY_VALIDITY_SCHEDULES);
+
+							if (StringUtils.isNotEmpty(validitySchedulesStr)) {
+								List<RangerValiditySchedule> validitySchedules = JsonUtils.jsonToRangerValiditySchedule(validitySchedulesStr);
+
+								ret.setValiditySchedules(validitySchedules);
+							}
+						}
+
+						getPolicyLabels(ret);
+						getResource(ret);
+						getPolicyItems(ret);
+					}
+				}
+
+				return ret;
+			}
+
+            private void getPolicyLabels(RangerPolicy ret) {
+                List<String> xPolicyLabels = new ArrayList<String>();
+                while (iterPolicyLabels.hasNext()) {
+                    XXPolicyLabelMap xPolicyLabel = iterPolicyLabels.next();
+                    if (xPolicyLabel.getPolicyId().equals(ret.getId())) {
+                        String policyLabel = lookupCache.getPolicyLabelName(xPolicyLabel.getPolicyLabelId());
+                        if (policyLabel != null) {
+                            xPolicyLabels.add(policyLabel);
+                        }
+                        ret.setPolicyLabels(xPolicyLabels);
+                    } else {
+                        if (iterPolicyLabels.hasPrevious()) {
+                            iterPolicyLabels.previous();
+                        }
+                        break;
+                    }
+                }
+            }
+
+			List<RangerPolicy> getAllPolicies() {
+				List<RangerPolicy> ret = new ArrayList<RangerPolicy>();
+
+				while (iterPolicy.hasNext()) {
+					RangerPolicy policy = getNextPolicy();
+
+					if (policy != null) {
+						ret.add(policy);
+					}
+				}
+
+				if (!hasProcessedAll()) {
+					LOG.warn("getAllPolicies(): perhaps one or more policies got updated during retrieval. Falling back to secondary method");
+
+					ret = getAllPoliciesBySecondary();
+				}
+
+				return ret;
+			}
+
+			List<RangerPolicy> getAllPoliciesBySecondary() {
+				List<RangerPolicy> ret = null;
+
+				if (service != null) {
+					List<XXPolicy> xPolicies = daoMgr.getXXPolicy().findByServiceId(service.getId());
+
+					if (CollectionUtils.isNotEmpty(xPolicies)) {
+						ret = new ArrayList<RangerPolicy>(xPolicies.size());
+
+						for (XXPolicy xPolicy : xPolicies) {
+							RetrieverContext ctx = new RetrieverContext(xPolicy, service);
+
+							RangerPolicy policy = ctx.getNextPolicy();
+
+							if (policy != null) {
+								ret.add(policy);
+							}
+						}
+					}
+				}
+
+				return ret;
+			}
+
+			private boolean hasProcessedAll() {
+				boolean moreToProcess = iterPolicy.hasNext() || iterResources.hasNext() || iterResourceMaps.hasNext()
+						|| iterPolicyItems.hasNext() || iterUserPerms.hasNext() || iterGroupPerms.hasNext()
+						|| iterAccesses.hasNext() || iterConditions.hasNext() || iterDataMaskInfos.hasNext()
+						|| iterRowFilterInfos.hasNext() || iterPolicyLabels.hasNext();
+
+				return !moreToProcess;
+			}
+
+			private void getResource(RangerPolicy policy) {
+				while (iterResources.hasNext()) {
+					XXPolicyResource xResource = iterResources.next();
+
+					if (xResource.getPolicyid().equals(policy.getId())) {
+						RangerPolicyResource resource = new RangerPolicyResource();
+
+						resource.setIsExcludes(xResource.getIsexcludes());
+						resource.setIsRecursive(xResource.getIsrecursive());
+
+						while (iterResourceMaps.hasNext()) {
+							XXPolicyResourceMap xResourceMap = iterResourceMaps.next();
+
+							if (xResourceMap.getResourceid().equals(xResource.getId())) {
+								resource.getValues().add(xResourceMap.getValue());
+							} else {
+								if (iterResourceMaps.hasPrevious()) {
+									iterResourceMaps.previous();
+								}
+
+								break;
+							}
+						}
+
+						policy.getResources().put(lookupCache.getResourceName(xResource.getResdefid()), resource);
+					} else if (xResource.getPolicyid().compareTo(policy.getId()) > 0) {
+						if (iterResources.hasPrevious()) {
+							iterResources.previous();
+						}
+
+						break;
+					}
+				}
+			}
+
+			private void getPolicyItems(RangerPolicy policy) {
+				while (iterPolicyItems.hasNext()) {
+					XXPolicyItem xPolicyItem = iterPolicyItems.next();
+
+					if (xPolicyItem.getPolicyid().equals(policy.getId())) {
+						final RangerPolicyItem          policyItem;
+						final RangerDataMaskPolicyItem  dataMaskPolicyItem;
+						final RangerRowFilterPolicyItem rowFilterPolicyItem;
+
+						if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) {
+							dataMaskPolicyItem  = new RangerDataMaskPolicyItem();
+							rowFilterPolicyItem = null;
+							policyItem          = dataMaskPolicyItem;
+						} else if (xPolicyItem.getItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) {
+							dataMaskPolicyItem  = null;
+							rowFilterPolicyItem = new RangerRowFilterPolicyItem();
+							policyItem          = rowFilterPolicyItem;
+						} else {
+							dataMaskPolicyItem  = null;
+							rowFilterPolicyItem = null;
+							policyItem          = new RangerPolicyItem();
+						}
+
+						while (iterAccesses.hasNext()) {
+							XXPolicyItemAccess xAccess = iterAccesses.next();
+
+							if (xAccess.getPolicyitemid().equals(xPolicyItem.getId())) {
+								policyItem.getAccesses().add(new RangerPolicyItemAccess(lookupCache.getAccessType(xAccess.getType()), xAccess.getIsallowed()));
+							} else {
+								if (iterAccesses.hasPrevious()) {
+									iterAccesses.previous();
+								}
+
+								break;
+							}
+						}
+
+						while (iterUserPerms.hasNext()) {
+							XXPolicyItemUserPerm xUserPerm = iterUserPerms.next();
+
+							if (xUserPerm.getPolicyitemid().equals(xPolicyItem.getId())) {
+								String userName = lookupCache.getUserName(xUserPerm.getUserid());
+
+								if (userName != null) {
+									policyItem.getUsers().add(userName);
+								}
+							} else {
+								if (iterUserPerms.hasPrevious()) {
+									iterUserPerms.previous();
+								}
+
+								break;
+							}
+						}
+
+						while (iterGroupPerms.hasNext()) {
+							XXPolicyItemGroupPerm xGroupPerm = iterGroupPerms.next();
+
+							if (xGroupPerm.getPolicyitemid().equals(xPolicyItem.getId())) {
+								String groupName = lookupCache.getGroupName(xGroupPerm.getGroupid());
+
+								if (groupName != null) {
+									policyItem.getGroups().add(groupName);
+								}
+							} else {
+								if (iterGroupPerms.hasPrevious()) {
+									iterGroupPerms.previous();
+								}
+
+								break;
+							}
+						}
+
+						RangerPolicyItemCondition condition         = null;
+						Long                      prevConditionType = null;
+
+						while (iterConditions.hasNext()) {
+							XXPolicyItemCondition xCondition = iterConditions.next();
+
+							if (xCondition.getPolicyitemid().equals(xPolicyItem.getId())) {
+								if (!xCondition.getType().equals(prevConditionType)) {
+									condition = new RangerPolicyItemCondition();
+
+									condition.setType(lookupCache.getConditionType(xCondition.getType()));
+									condition.getValues().add(xCondition.getValue());
+
+									policyItem.getConditions().add(condition);
+
+									prevConditionType = xCondition.getType();
+								} else {
+									condition.getValues().add(xCondition.getValue());
+								}
+							} else {
+								if (iterConditions.hasPrevious()) {
+									iterConditions.previous();
+								}
+
+								break;
+							}
+						}
+
+						policyItem.setDelegateAdmin(xPolicyItem.getDelegateAdmin());
+
+						if (dataMaskPolicyItem != null) {
+							while (iterDataMaskInfos.hasNext()) {
+								XXPolicyItemDataMaskInfo xDataMaskInfo = iterDataMaskInfos.next();
+
+								if (xDataMaskInfo.getPolicyItemId().equals(xPolicyItem.getId())) {
+									dataMaskPolicyItem.setDataMaskInfo(new RangerPolicyItemDataMaskInfo(lookupCache.getDataMaskName(xDataMaskInfo.getType()), xDataMaskInfo.getConditionExpr(), xDataMaskInfo.getValueExpr()));
+								} else {
+									if (iterDataMaskInfos.hasPrevious()) {
+										iterDataMaskInfos.previous();
+									}
+
+									break;
+								}
+							}
+						}
+
+						if (rowFilterPolicyItem != null) {
+							while (iterRowFilterInfos.hasNext()) {
+								XXPolicyItemRowFilterInfo xRowFilterInfo = iterRowFilterInfos.next();
+
+								if (xRowFilterInfo.getPolicyItemId().equals(xPolicyItem.getId())) {
+									rowFilterPolicyItem.setRowFilterInfo(new RangerPolicyItemRowFilterInfo(xRowFilterInfo.getFilterExpr()));
+								} else {
+									if (iterRowFilterInfos.hasPrevious()) {
+										iterRowFilterInfos.previous();
+									}
+
+									break;
+								}
+							}
+						}
+
+						int itemType = xPolicyItem.getItemType() == null ? RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW : xPolicyItem.getItemType();
+
+						if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) {
+							policy.getPolicyItems().add(policyItem);
+						} else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY) {
+							policy.getDenyPolicyItems().add(policyItem);
+						} else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW_EXCEPTIONS) {
+							policy.getAllowExceptions().add(policyItem);
+						} else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DENY_EXCEPTIONS) {
+							policy.getDenyExceptions().add(policyItem);
+						} else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_DATAMASK) {
+							policy.getDataMaskPolicyItems().add(dataMaskPolicyItem);
+						} else if (itemType == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ROWFILTER) {
+							policy.getRowFilterPolicyItems().add(rowFilterPolicyItem);
+						} else { // unknown itemType
+							LOG.warn("RangerPolicyRetriever.getPolicy(policyId=" + policy.getId() + "): ignoring unknown policyItemType " + itemType);
+						}
+					} else if (xPolicyItem.getPolicyid().compareTo(policy.getId()) > 0) {
+						if (iterPolicyItems.hasPrevious()) {
+							iterPolicyItems.previous();
+						}
+
+						break;
+					}
+				}
+			}
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/ranger/blob/48cfb403/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java
new file mode 100644
index 0000000..035e023
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java
@@ -0,0 +1,789 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *	 http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Consolidates Ranger policy details into a JSON string and stores it into a
+ * column in x_policy table After running this patch Ranger policy can be
+ * completely read/saved into x_policy table and some related Ref tables (which
+ * maintain ID->String mapping for each policy).
+ *
+ */
+
+package org.apache.ranger.patch;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Logger;
+import org.apache.ranger.authorization.utils.JsonUtils;
+import org.apache.ranger.authorization.utils.StringUtil;
+import org.apache.ranger.biz.ServiceDBStore;
+import org.apache.ranger.biz.TagDBStore;
+import org.apache.ranger.db.RangerDaoManager;
+import org.apache.ranger.entity.XXPortalUser;
+import org.apache.ranger.entity.XXResourceDef;
+import org.apache.ranger.entity.XXService;
+import org.apache.ranger.entity.XXServiceResource;
+import org.apache.ranger.entity.XXServiceResourceElement;
+import org.apache.ranger.entity.XXServiceResourceElementValue;
+import org.apache.ranger.entity.XXTag;
+import org.apache.ranger.entity.XXTagAttribute;
+import org.apache.ranger.entity.XXTagAttributeDef;
+import org.apache.ranger.entity.XXTagDef;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.plugin.model.RangerTag;
+import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.model.RangerValiditySchedule;
+import org.apache.ranger.plugin.util.SearchFilter;
+import org.apache.ranger.service.RangerServiceResourceService;
+import org.apache.ranger.service.RangerTagDefService;
+import org.apache.ranger.service.RangerTagService;
+import org.apache.ranger.util.CLIUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
+
+@Component
+public class PatchForUpdatingTagsJson_J10020 extends BaseLoader {
+
+    private static final Logger logger = Logger.getLogger(PatchForUpdatingTagsJson_J10020.class);
+
+    @Autowired
+    RangerDaoManager daoMgr;
+
+    @Autowired
+    ServiceDBStore svcStore;
+
+    @Autowired
+    TagDBStore tagStore;
+
+    @Autowired
+    @Qualifier(value = "transactionManager")
+    PlatformTransactionManager txManager;
+
+    @Autowired
+    RangerTagDefService tagDefService;
+
+    @Autowired
+    RangerTagService tagService;
+
+    @Autowired
+    RangerServiceResourceService serviceResourceService;
+
+    public static void main(String[] args) {
+        logger.info("main()");
+        try {
+            PatchForUpdatingTagsJson_J10020 loader = (PatchForUpdatingTagsJson_J10020) CLIUtil
+                    .getBean(PatchForUpdatingTagsJson_J10020.class);
+
+            loader.init();
+
+            while (loader.isMoreToProcess()) {
+                loader.load();
+            }
+
+            logger.info("Load complete. Exiting!!!");
+
+            System.exit(0);
+        } catch (Exception e) {
+            logger.error("Error loading", e);
+            System.exit(1);
+        }
+    }
+
+    @Override
+    public void init() throws Exception {
+        // Do Nothing
+    }
+
+    @Override
+    public void execLoad() {
+        logger.info("==> PatchForUpdatingTagsJson.execLoad()");
+
+        try {
+            updateRangerTagsTablesWithTagsJson();
+        } catch (Exception e) {
+            logger.error("Error while UpdateRangerTagsTablesWithTagsJson()", e);
+            System.exit(1);
+        }
+
+        logger.info("<== PatchForUpdatingTagsJson.execLoad()");
+    }
+
+    @Override
+    public void printStats() {
+        logger.info("Update Ranger Tags Tables with Json data ");
+    }
+
+    private void updateRangerTagsTablesWithTagsJson() throws Exception {
+        logger.info("==> updateRangerTagsTablesWithTagsJson() ");
+
+        List<RangerService> allServices = svcStore.getServices(new SearchFilter());
+
+        if (CollectionUtils.isNotEmpty(allServices)) {
+            for (RangerService service : allServices) {
+                XXService                   dbService        = daoMgr.getXXService().getById(service.getId());
+                RangerTagDBRetriever        tagsRetriever    = new RangerTagDBRetriever(daoMgr, txManager, dbService);
+                Map<Long, RangerTagDef>     tagDefs          = tagsRetriever.getTagDefs();
+                Map<Long, RangerTag>        tags             = tagsRetriever.getTags();
+                List<RangerServiceResource> serviceResources = tagsRetriever.getServiceResources();
+
+                if (MapUtils.isNotEmpty(tagDefs)) {
+                    logger.info("==> Port " + tagDefs.size() + " Tag Definitions for service(name=" + dbService.getName() + ")");
+
+                    for (Map.Entry<Long, RangerTagDef> entry : tagDefs.entrySet()) {
+                        RangerTagDef tagDef = entry.getValue();
+
+                        portTagDef(tagDef);
+                    }
+                }
+
+                if (MapUtils.isNotEmpty(tags)) {
+                    logger.info("==> Port " + tags.size() + " Tags for service(name=" + dbService.getName() + ")");
+
+                    for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) {
+                        RangerTag tag = entry.getValue();
+
+                        portTag(tag);
+                    }
+                }
+
+                if (CollectionUtils.isNotEmpty(serviceResources)) {
+                    logger.info("==> Port " + serviceResources.size() + " Service Resources for service(name=" + dbService.getName() + ")");
+
+                    for (RangerServiceResource serviceResource : serviceResources) {
+                        portServiceResource(serviceResource);
+                    }
+                }
+            }
+        }
+
+        logger.info("<== updateRangerTagsTablesWithTagsJson() ");
+    }
+
+    private void portTagDef(RangerTagDef tagDef) {
+        tagDefService.update(tagDef);
+    }
+
+    private void portTag(RangerTag tag) {
+        tagService.update(tag);
+    }
+
+    private void portServiceResource(RangerServiceResource serviceResource) throws Exception {
+        serviceResourceService.update(serviceResource);
+        tagStore.refreshServiceResource(serviceResource.getId());
+    }
+
+    private class RangerTagDBRetriever {
+        final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class);
+
+        private final RangerDaoManager                 daoMgr;
+        private final XXService                        xService;
+        private final RangerTagDBRetriever.LookupCache lookupCache;
+        private final PlatformTransactionManager       txManager;
+        private final TransactionTemplate              txTemplate;
+        private       List<RangerServiceResource>      serviceResources;
+        private       Map<Long, RangerTagDef>          tagDefs;
+        private       Map<Long, RangerTag>             tags;
+
+        RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) throws InterruptedException {
+            this.daoMgr      = daoMgr;
+            this.xService    = xService;
+            this.lookupCache = new RangerTagDBRetriever.LookupCache();
+            this.txManager   = txManager;
+
+            if (this.txManager != null) {
+                this.txTemplate = new TransactionTemplate(this.txManager);
+                this.txTemplate.setReadOnly(true);
+            } else {
+                this.txTemplate = null;
+            }
+
+            if (this.daoMgr != null && this.xService != null) {
+                if (this.txTemplate == null) {
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("Load Tags in the same thread and using an existing transaction");
+                    }
+
+                    if (!initializeTagCache(xService)) {
+                        LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction");
+                    }
+                } else {
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("Load Tags in a separate thread and using a new transaction");
+                    }
+
+                    RangerTagDBRetriever.TagLoaderThread t = new RangerTagDBRetriever.TagLoaderThread(txTemplate, xService);
+                    t.setDaemon(true);
+                    t.start();
+                    t.join();
+                }
+
+            }
+        }
+
+        List<RangerServiceResource> getServiceResources() {
+            return serviceResources;
+        }
+
+        Map<Long, RangerTagDef> getTagDefs() {
+            return tagDefs;
+        }
+
+        Map<Long, RangerTag> getTags() {
+            return tags;
+        }
+
+        private boolean initializeTagCache(XXService xService) {
+            boolean ret;
+            RangerTagDBRetriever.TagRetrieverServiceResourceContext serviceResourceContext  = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xService);
+            RangerTagDBRetriever.TagRetrieverTagDefContext          tagDefContext           = new RangerTagDBRetriever.TagRetrieverTagDefContext(xService);
+            RangerTagDBRetriever.TagRetrieverTagContext             tagContext              = new RangerTagDBRetriever.TagRetrieverTagContext(xService);
+
+            serviceResources = serviceResourceContext.getAllServiceResources();
+            tagDefs          = tagDefContext.getAllTagDefs();
+            tags             = tagContext.getAllTags();
+
+            ret = true;
+            return ret;
+        }
+
+        private <T> List<T> asList(T obj) {
+            List<T> ret = new ArrayList<>();
+
+            if (obj != null) {
+                ret.add(obj);
+            }
+
+            return ret;
+        }
+
+        private class LookupCache {
+            final Map<Long, String> userScreenNames = new HashMap<>();
+            final Map<Long, String> resourceDefs    = new HashMap<>();
+
+            String getUserScreenName(Long userId) {
+                String ret = null;
+
+                if (userId != null) {
+                    ret = userScreenNames.get(userId);
+
+                    if (ret == null) {
+                        XXPortalUser user = daoMgr.getXXPortalUser().getById(userId);
+
+                        if (user != null) {
+                            ret = user.getPublicScreenName();
+
+                            if (StringUtil.isEmpty(ret)) {
+                                ret = user.getFirstName();
+
+                                if (StringUtil.isEmpty(ret)) {
+                                    ret = user.getLoginId();
+                                } else {
+                                    if (!StringUtil.isEmpty(user.getLastName())) {
+                                        ret += (" " + user.getLastName());
+                                    }
+                                }
+                            }
+
+                            if (ret != null) {
+                                userScreenNames.put(userId, ret);
+                            }
+                        }
+                    }
+                }
+
+                return ret;
+            }
+
+            String getResourceName(Long resourceDefId) {
+                String ret = null;
+
+                if (resourceDefId != null) {
+                    ret = resourceDefs.get(resourceDefId);
+
+                    if (ret == null) {
+                        XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId);
+
+                        if (xResourceDef != null) {
+                            ret = xResourceDef.getName();
+
+                            resourceDefs.put(resourceDefId, ret);
+                        }
+                    }
+                }
+
+                return ret;
+            }
+        }
+
+        private class TagLoaderThread extends Thread {
+            final TransactionTemplate txTemplate;
+            final XXService           xService;
+
+            TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+                this.txTemplate = txTemplate;
+                this.xService   = xService;
+            }
+
+            @Override
+            public void run() {
+                txTemplate.setReadOnly(true);
+                Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+                    @Override
+                    public Boolean doInTransaction(TransactionStatus status) {
+                        boolean ret = initializeTagCache(xService);
+
+                        if (!ret) {
+                            status.setRollbackOnly();
+                            LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction");
+                        }
+                        return ret;
+                    }
+                });
+
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("transaction result:[" + result +"]");
+                }
+            }
+        }
+
+        private class TagRetrieverServiceResourceContext {
+            final XXService                                   service;
+            final ListIterator<XXServiceResource>             iterServiceResource;
+            final ListIterator<XXServiceResourceElement>      iterServiceResourceElement;
+            final ListIterator<XXServiceResourceElementValue> iterServiceResourceElementValue;
+
+            TagRetrieverServiceResourceContext(XXService xService) {
+                Long                                serviceId                     = xService == null ? null : xService.getId();
+                List<XXServiceResource>             xServiceResources             = daoMgr.getXXServiceResource().findByServiceId(serviceId);
+                List<XXServiceResourceElement>      xServiceResourceElements      = daoMgr.getXXServiceResourceElement().findTaggedResourcesInServiceId(serviceId);
+                List<XXServiceResourceElementValue> xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findTaggedResourcesInServiceId(serviceId);
+
+                this.service                         = xService;
+                this.iterServiceResource             = xServiceResources.listIterator();
+                this.iterServiceResourceElement      = xServiceResourceElements.listIterator();
+                this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator();
+
+            }
+
+            TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) {
+                Long                                resourceId                    = xServiceResource == null ? null : xServiceResource.getId();
+                List<XXServiceResource>             xServiceResources             = asList(xServiceResource);
+                List<XXServiceResourceElement>      xServiceResourceElements      = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId);
+                List<XXServiceResourceElementValue> xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId);
+
+                this.service                         = xService;
+                this.iterServiceResource             = xServiceResources.listIterator();
+                this.iterServiceResourceElement      = xServiceResourceElements.listIterator();
+                this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator();
+            }
+
+            List<RangerServiceResource> getAllServiceResources() {
+                List<RangerServiceResource> ret = new ArrayList<>();
+
+                while (iterServiceResource.hasNext()) {
+                    RangerServiceResource serviceResource = getNextServiceResource();
+
+                    if (serviceResource != null) {
+                        ret.add(serviceResource);
+                    }
+                }
+
+                if (!hasProcessedAll()) {
+                    LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... ");
+
+                    ret = getServiceResourcesBySecondary();
+                }
+
+                return ret;
+            }
+
+            RangerServiceResource getNextServiceResource() {
+                RangerServiceResource ret = null;
+
+                if (iterServiceResource.hasNext()) {
+                    XXServiceResource xServiceResource = iterServiceResource.next();
+
+                    if (xServiceResource != null) {
+                        ret = new RangerServiceResource();
+
+                        ret.setId(xServiceResource.getId());
+                        ret.setGuid(xServiceResource.getGuid());
+                        ret.setIsEnabled(xServiceResource.getIsEnabled());
+                        ret.setCreatedBy(lookupCache.getUserScreenName(xServiceResource.getAddedByUserId()));
+                        ret.setUpdatedBy(lookupCache.getUserScreenName(xServiceResource.getUpdatedByUserId()));
+                        ret.setCreateTime(xServiceResource.getCreateTime());
+                        ret.setUpdateTime(xServiceResource.getUpdateTime());
+                        ret.setVersion(xServiceResource.getVersion());
+                        ret.setResourceSignature(xServiceResource.getResourceSignature());
+                        ret.setServiceName(xService.getName());
+
+                        getServiceResourceElements(ret);
+                    }
+                }
+
+                return ret;
+            }
+
+            void getServiceResourceElements(RangerServiceResource serviceResource) {
+                while (iterServiceResourceElement.hasNext()) {
+                    XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next();
+
+                    if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) {
+                        RangerPolicy.RangerPolicyResource resource = new RangerPolicy.RangerPolicyResource();
+
+                        resource.setIsExcludes(xServiceResourceElement.getIsExcludes());
+                        resource.setIsRecursive(xServiceResourceElement.getIsRecursive());
+
+                        while (iterServiceResourceElementValue.hasNext()) {
+                            XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next();
+
+                            if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) {
+                                resource.getValues().add(xServiceResourceElementValue.getValue());
+                            } else {
+                                if (iterServiceResourceElementValue.hasPrevious()) {
+                                    iterServiceResourceElementValue.previous();
+                                }
+
+                                break;
+                            }
+                        }
+
+                        serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource);
+                    } else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) {
+                        if (iterServiceResourceElement.hasPrevious()) {
+                            iterServiceResourceElement.previous();
+                        }
+
+                        break;
+                    }
+                }
+            }
+
+            boolean hasProcessedAll() {
+                boolean moreToProcess = iterServiceResource.hasNext()
+                        || iterServiceResourceElement.hasNext()
+                        || iterServiceResourceElementValue.hasNext();
+
+                return !moreToProcess;
+            }
+
+            List<RangerServiceResource> getServiceResourcesBySecondary() {
+                List<RangerServiceResource> ret = null;
+
+                if (service != null) {
+                    List<XXServiceResource> xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(service.getId());
+
+                    if (CollectionUtils.isNotEmpty(xServiceResources)) {
+                        ret = new ArrayList<>(xServiceResources.size());
+
+                        for (XXServiceResource xServiceResource : xServiceResources) {
+                            RangerTagDBRetriever.TagRetrieverServiceResourceContext ctx = new RangerTagDBRetriever.TagRetrieverServiceResourceContext(xServiceResource, service);
+
+                            RangerServiceResource serviceResource = ctx.getNextServiceResource();
+
+                            if (serviceResource != null) {
+                                ret.add(serviceResource);
+                            }
+                        }
+                    }
+                }
+                return ret;
+            }
+        }
+
+        private class TagRetrieverTagDefContext {
+            final XXService                       service;
+            final ListIterator<XXTagDef>          iterTagDef;
+            final ListIterator<XXTagAttributeDef> iterTagAttributeDef;
+
+
+            TagRetrieverTagDefContext(XXService xService) {
+                Long                    serviceId         = xService == null ? null : xService.getId();
+                List<XXTagDef>          xTagDefs          = daoMgr.getXXTagDef().findByServiceId(serviceId);
+                List<XXTagAttributeDef> xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByServiceId(serviceId);
+
+                this.service             = xService;
+                this.iterTagDef          = xTagDefs.listIterator();
+                this.iterTagAttributeDef = xTagAttributeDefs.listIterator();
+            }
+
+            TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) {
+                Long                    tagDefId          = xTagDef == null ? null : xTagDef.getId();
+                List<XXTagDef>          xTagDefs          = asList(xTagDef);
+                List<XXTagAttributeDef> xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId);
+
+                this.service             = xService;
+                this.iterTagDef          = xTagDefs.listIterator();
+                this.iterTagAttributeDef = xTagAttributeDefs.listIterator();
+            }
+
+            Map<Long, RangerTagDef> getAllTagDefs() {
+                Map<Long, RangerTagDef> ret = new HashMap<>();
+
+                while (iterTagDef.hasNext()) {
+                    RangerTagDef tagDef = getNextTagDef();
+
+                    if (tagDef != null) {
+                        ret.put(tagDef.getId(), tagDef);
+                    }
+                }
+
+                if (!hasProcessedAllTagDefs()) {
+                    LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval.  Using fallback ... ");
+
+                    ret = getTagDefsBySecondary();
+                }
+
+                return ret;
+            }
+
+            RangerTagDef getNextTagDef() {
+                RangerTagDef ret = null;
+
+                if (iterTagDef.hasNext()) {
+                    XXTagDef xTagDef = iterTagDef.next();
+
+                    if (xTagDef != null) {
+                        ret = new RangerTagDef();
+
+                        ret.setId(xTagDef.getId());
+                        ret.setGuid(xTagDef.getGuid());
+                        ret.setIsEnabled(xTagDef.getIsEnabled());
+                        ret.setCreatedBy(lookupCache.getUserScreenName(xTagDef.getAddedByUserId()));
+                        ret.setUpdatedBy(lookupCache.getUserScreenName(xTagDef.getUpdatedByUserId()));
+                        ret.setCreateTime(xTagDef.getCreateTime());
+                        ret.setUpdateTime(xTagDef.getUpdateTime());
+                        ret.setVersion(xTagDef.getVersion());
+                        ret.setName(xTagDef.getName());
+                        ret.setSource(xTagDef.getSource());
+
+                        getTagAttributeDefs(ret);
+                    }
+                }
+
+                return ret;
+            }
+
+            void getTagAttributeDefs(RangerTagDef tagDef) {
+                while (iterTagAttributeDef.hasNext()) {
+                    XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next();
+
+                    if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) {
+                        RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef();
+
+                        tagAttributeDef.setName(xTagAttributeDef.getName());
+                        tagAttributeDef.setType(xTagAttributeDef.getType());
+
+                        tagDef.getAttributeDefs().add(tagAttributeDef);
+                    } else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) {
+                        if (iterTagAttributeDef.hasPrevious()) {
+                            iterTagAttributeDef.previous();
+                        }
+                        break;
+                    }
+                }
+            }
+
+            boolean hasProcessedAllTagDefs() {
+                boolean moreToProcess = iterTagAttributeDef.hasNext();
+
+                return !moreToProcess;
+            }
+
+            Map<Long, RangerTagDef> getTagDefsBySecondary() {
+                Map<Long, RangerTagDef> ret = null;
+
+                if (service != null) {
+                    List<XXTagDef> xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId());
+
+                    if (CollectionUtils.isNotEmpty(xTagDefs)) {
+                        ret = new HashMap<>(xTagDefs.size());
+
+                        for (XXTagDef xTagDef : xTagDefs) {
+                            TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service);
+
+                            RangerTagDef tagDef = ctx.getNextTagDef();
+
+                            if (tagDef != null) {
+                                ret.put(tagDef.getId(), tagDef);
+                            }
+                        }
+                    }
+                }
+                return ret;
+            }
+        }
+
+        private class TagRetrieverTagContext {
+            final XXService                    service;
+            final ListIterator<XXTag>          iterTag;
+            final ListIterator<XXTagAttribute> iterTagAttribute;
+
+
+            TagRetrieverTagContext(XXService xService) {
+                Long                 serviceId      = xService == null ? null : xService.getId();
+                List<XXTag>          xTags          = daoMgr.getXXTag().findByServiceId(serviceId);
+                List<XXTagAttribute> xTagAttributes = daoMgr.getXXTagAttribute().findByServiceId(serviceId);
+
+                this.service          = xService;
+                this.iterTag          = xTags.listIterator();
+                this.iterTagAttribute = xTagAttributes.listIterator();
+
+            }
+
+            TagRetrieverTagContext(XXTag xTag, XXService xService) {
+                Long                 tagId          = xTag == null ? null : xTag.getId();
+                List<XXTag>          xTags          = asList(xTag);
+                List<XXTagAttribute> xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId);
+
+                this.service          = xService;
+                this.iterTag          = xTags.listIterator();
+                this.iterTagAttribute = xTagAttributes.listIterator();
+            }
+
+
+            Map<Long, RangerTag> getAllTags() {
+                Map<Long, RangerTag> ret = new HashMap<>();
+
+                while (iterTag.hasNext()) {
+                    RangerTag tag = getNextTag();
+
+                    if (tag != null) {
+                        ret.put(tag.getId(), tag);
+                    }
+                }
+
+                if (!hasProcessedAllTags()) {
+                    LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... ");
+
+                    ret = getTagsBySecondary();
+                }
+
+                return ret;
+            }
+
+            RangerTag getNextTag() {
+                RangerTag ret = null;
+
+                if (iterTag.hasNext()) {
+                    XXTag xTag = iterTag.next();
+
+                    if (xTag != null) {
+                        ret = new RangerTag();
+
+                        ret.setId(xTag.getId());
+                        ret.setGuid(xTag.getGuid());
+                        ret.setOwner(xTag.getOwner());
+                        ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId()));
+                        ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId()));
+                        ret.setCreateTime(xTag.getCreateTime());
+                        ret.setUpdateTime(xTag.getUpdateTime());
+                        ret.setVersion(xTag.getVersion());
+
+                        Map<String, String> mapOfOptions = JsonUtils.jsonToMapStringString(xTag.getOptions());
+
+                        if (MapUtils.isNotEmpty(mapOfOptions)) {
+                            String validityPeriodsStr = mapOfOptions.get(RangerTag.OPTION_TAG_VALIDITY_PERIODS);
+
+                            if (StringUtils.isNotEmpty(validityPeriodsStr)) {
+                                List<RangerValiditySchedule> validityPeriods = JsonUtils.jsonToRangerValiditySchedule(validityPeriodsStr);
+
+                                ret.setValidityPeriods(validityPeriods);
+                            }
+                        }
+
+                        Map<Long, RangerTagDef> tagDefs = getTagDefs();
+                        if (tagDefs != null) {
+                            RangerTagDef tagDef = tagDefs.get(xTag.getType());
+                            if (tagDef != null) {
+                                ret.setType(tagDef.getName());
+                            }
+                        }
+
+                        getTagAttributes(ret);
+                    }
+                }
+
+                return ret;
+            }
+
+            void getTagAttributes(RangerTag tag) {
+                while (iterTagAttribute.hasNext()) {
+                    XXTagAttribute xTagAttribute = iterTagAttribute.next();
+
+                    if (xTagAttribute.getTagId().equals(tag.getId())) {
+                        String attributeName = xTagAttribute.getName();
+                        String attributeValue = xTagAttribute.getValue();
+
+                        tag.getAttributes().put(attributeName, attributeValue);
+                    } else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) {
+                        if (iterTagAttribute.hasPrevious()) {
+                            iterTagAttribute.previous();
+                        }
+                        break;
+                    }
+                }
+            }
+
+            boolean hasProcessedAllTags() {
+                boolean moreToProcess = iterTagAttribute.hasNext();
+
+                return !moreToProcess;
+            }
+
+            Map<Long, RangerTag> getTagsBySecondary() {
+                Map<Long, RangerTag> ret = null;
+
+                if (service != null) {
+                    List<XXTag> xTags = daoMgr.getXXTag().findByServiceId(service.getId());
+
+                    if (CollectionUtils.isNotEmpty(xTags)) {
+                        ret = new HashMap<>(xTags.size());
+
+                        for (XXTag xTag : xTags) {
+                            TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service);
+
+                            RangerTag tag = ctx.getNextTag();
+
+                            if (tag != null) {
+                                ret.put(tag.getId(), tag);
+                            }
+                        }
+                    }
+                }
+                return ret;
+            }
+        }
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/ranger/blob/48cfb403/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index 97b8d71..c513548 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -101,8 +101,8 @@ import org.apache.ranger.plugin.policyengine.RangerPolicyEngineCacheForEngineOpt
 import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl;
 import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
 import org.apache.ranger.plugin.service.ResourceLookupContext;
-import org.apache.ranger.plugin.store.PList;
 import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
+import org.apache.ranger.plugin.store.PList;
 import org.apache.ranger.plugin.store.ServiceStore;
 import org.apache.ranger.plugin.util.GrantRevokeRequest;
 import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
@@ -1515,10 +1515,11 @@ public class ServiceREST {
 				String serviceName    = request.getParameter(PARAM_SERVICE_NAME);
 				String policyName     = request.getParameter(PARAM_POLICY_NAME);
 				String updateIfExists = request.getParameter(PARAM_UPDATE_IF_EXISTS);
-				
-				if (serviceName == null && policyName == null && updateIfExists != null && updateIfExists.equalsIgnoreCase("true")){
-					serviceName    = (String) request.getAttribute(PARAM_SERVICE_NAME);
-					policyName     = (String) request.getAttribute(PARAM_POLICY_NAME);
+
+				if (serviceName == null && policyName == null && updateIfExists != null
+						&& updateIfExists.equalsIgnoreCase("true")) {
+					serviceName = (String) request.getAttribute(PARAM_SERVICE_NAME);
+					policyName = (String) request.getAttribute(PARAM_POLICY_NAME);
 				}
 
 				if(StringUtils.isNotEmpty(serviceName)) {
@@ -1529,7 +1530,7 @@ public class ServiceREST {
 					policy.setName(StringUtils.trim(policyName));
 				}
 
-                                if(updateIfExists != null && Boolean.valueOf(updateIfExists)) {
+				if (updateIfExists != null && Boolean.valueOf(updateIfExists)) {
 					RangerPolicy existingPolicy = null;
 					try {
 						if(StringUtils.isNotEmpty(policy.getGuid())) {
@@ -2413,7 +2414,7 @@ public class ServiceREST {
 										validator.validate(rangerPolicy.getId(), Action.DELETE);
 										ensureAdminAccess(rangerPolicy);
                                                                                 bizUtil.blockAuditorRoleUser();
-										svcStore.deletePolicy(rangerPolicy);
+										svcStore.deletePolicy(rangerPolicy.getId());
 										totalDeletedPilicies = totalDeletedPilicies + 1;
 										if (LOG.isDebugEnabled()) {
 											LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully." );
@@ -3149,7 +3150,7 @@ public class ServiceREST {
 								ret.addAll(listToFilter);
 							}
 						} else if (isServiceAdminUser) {
-                                ret.addAll(listToFilter);
+							ret.addAll(listToFilter);
 						}
 
 						continue;
@@ -3552,4 +3553,4 @@ public class ServiceREST {
                         }
                 }
         }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ranger/blob/48cfb403/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java
index e6a519a..fe1883d 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerAuditFields.java
@@ -23,7 +23,7 @@ import org.apache.ranger.entity.XXDBBase;
 import org.springframework.stereotype.Component;
 
 @Component
-public class RangerAuditFields {
+public class RangerAuditFields<T extends XXDBBase> {
 
 	public <T extends XXDBBase, PARENT extends XXDBBase> T populateAuditFields(T xObj, PARENT parentObj) {
 		xObj.setCreateTime(parentObj.getCreateTime());

http://git-wip-us.apache.org/repos/asf/ranger/blob/48cfb403/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
index 8d42165..6ab12ad 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
@@ -17,7 +17,6 @@
 
 package org.apache.ranger.service;
 
-import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.ranger.authorization.utils.JsonUtils;
 import org.apache.ranger.common.GUIDUtil;
@@ -30,12 +29,10 @@ import org.apache.ranger.common.SortField.SORT_ORDER;
 import org.apache.ranger.entity.XXPolicyBase;
 import org.apache.ranger.entity.XXService;
 import org.apache.ranger.plugin.model.RangerPolicy;
-import org.apache.ranger.plugin.model.RangerValiditySchedule;
 import org.apache.ranger.plugin.util.SearchFilter;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends RangerPolicy> extends
@@ -58,18 +55,20 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends
 				"XXService xSvc", "xSvc.id = obj.service"));
 		searchFields
 				.add(new SearchField(SearchFilter.IS_ENABLED, "obj.isEnabled", DATA_TYPE.BOOLEAN, SEARCH_TYPE.FULL));
-		searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL,
-				"XXPolicyResource xPolRes","obj.id=xPolRes.policyId"));
+		//might need updation
+		/*searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL,
+				"XXPolicyResource xPolRes","obj.id=xPolRes.policyId"));*/
 		searchFields.add(new SearchField(SearchFilter.POLICY_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL));
 		searchFields.add(new SearchField(SearchFilter.POLICY_NAME, "obj.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL));
 		searchFields.add(new SearchField(SearchFilter.GUID, "obj.guid", DATA_TYPE.STRING, SEARCH_TYPE.FULL));
 		searchFields.add(new SearchField(SearchFilter.USER, "xUser.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL,
-				"XXUser xUser, XXPolicyItem xPolItem, XXPolicyItemUserPerm userPerm", "obj.id = xPolItem.policyId "
-						+ "and userPerm.policyItemId = xPolItem.id and xUser.id = userPerm.userId"));
+				"XXUser xUser, XXPolicyRefUser refUser", "obj.id = refUser.policyId "
+						+ "and xUser.id = refUser.userId"));
 		searchFields.add(new SearchField(SearchFilter.GROUP, "xGrp.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL,
-				"XXGroup xGrp, XXPolicyItem xPolItem, XXPolicyItemGroupPerm grpPerm", "obj.id = xPolItem.policyId "
-						+ "and grpPerm.policyItemId = xPolItem.id and xGrp.id = grpPerm.groupId"));
-		searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING,
+				"XXGroup xGrp , XXPolicyRefGroup refGroup", "obj.id = refGroup.policyId "
+						+ "and xGrp.id = refGroup.groupId"));
+		//might need updation
+		/*searchFields.add(new SearchField(SearchFilter.POL_RESOURCE, "resMap.value", DATA_TYPE.STRING,
 				SEARCH_TYPE.PARTIAL, "XXPolicyResourceMap resMap, XXPolicyResource polRes",
 				"resMap.resourceId = polRes.id and polRes.policyId = obj.id"));
                 /*searchFields.add(new SearchField(SearchFilter.POLICY_LABELS_PARTIAL, "obj.label_name", DATA_TYPE.STRING,
@@ -118,9 +117,11 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends
 			options.remove(OPTION_POLICY_VALIDITY_SCHEDULES);
 		}
 
-        xObj.setOptions(JsonUtils.mapToJson(options));
+		xObj.setOptions(JsonUtils.mapToJson(options));
 
-        return xObj;
+		xObj.setPolicyText(JsonUtils.objectToJson(vObj));
+
+		return xObj;
 	}
 
 	@Override
@@ -137,20 +138,16 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends
 		vObj.setIsEnabled(xObj.getIsEnabled());
 		vObj.setIsAuditEnabled(xObj.getIsAuditEnabled());
 
-		Map<String, Object> options = JsonUtils.jsonToObject(xObj.getOptions(), Map.class);
-
-		if (MapUtils.isNotEmpty(options)) {
-			String optionPolicyValiditySchedule = (String)options.remove(OPTION_POLICY_VALIDITY_SCHEDULES);
+		String policyText = xObj.getPolicyText();
 
-			if (StringUtils.isNotBlank(optionPolicyValiditySchedule)) {
-				List<RangerValiditySchedule> validitySchedules = JsonUtils.jsonToRangerValiditySchedule(optionPolicyValiditySchedule);
+		RangerPolicy ret = JsonUtils.jsonToObject(policyText, RangerPolicy.class);
 
-				vObj.setValiditySchedules(validitySchedules);
-			}
+		if (ret != null) {
+			vObj.setOptions(ret.getOptions());
+			vObj.setValiditySchedules(ret.getValiditySchedules());
+			vObj.setPolicyLabels(ret.getPolicyLabels());
 		}
 
-		vObj.setOptions(options);
-
 		return vObj;
 	}
 }

http://git-wip-us.apache.org/repos/asf/ranger/blob/48cfb403/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java
index 5cbe47a..9b543ef 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceDefServiceBase.java
@@ -62,7 +62,7 @@ public abstract class RangerServiceDefServiceBase<T extends XXServiceDefBase, V
 	private static final String OPTION_RESOURCE_IS_VALID_LEAF            = "__isValidLeaf";
 
 	@Autowired
-	RangerAuditFields rangerAuditFields;
+	RangerAuditFields<?> rangerAuditFields;
 
 	@Autowired
 	JSONUtil jsonUtil;


Mime
View raw message