Return-Path: X-Original-To: apmail-ranger-commits-archive@www.apache.org Delivered-To: apmail-ranger-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 25B1317A11 for ; Mon, 8 Jun 2015 08:15:03 +0000 (UTC) Received: (qmail 9324 invoked by uid 500); 8 Jun 2015 08:15:03 -0000 Delivered-To: apmail-ranger-commits-archive@ranger.apache.org Received: (qmail 9298 invoked by uid 500); 8 Jun 2015 08:15:03 -0000 Mailing-List: contact commits-help@ranger.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ranger.incubator.apache.org Delivered-To: mailing list commits@ranger.incubator.apache.org Received: (qmail 9289 invoked by uid 99); 8 Jun 2015 08:15:03 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 08 Jun 2015 08:15:03 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,T_RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Mon, 08 Jun 2015 08:12:50 +0000 Received: (qmail 8832 invoked by uid 99); 8 Jun 2015 08:14:37 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 08 Jun 2015 08:14:37 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 06345DFAFF; Mon, 8 Jun 2015 08:14:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@ranger.incubator.apache.org Date: Mon, 08 Jun 2015 08:14:36 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/6] incubator-ranger git commit: RANGER-534: fix upgrade issue in policy migration X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-ranger Updated Branches: refs/heads/tag-policy f7ec8219d -> f360a3ba3 RANGER-534: fix upgrade issue in policy migration Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/104d1b89 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/104d1b89 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/104d1b89 Branch: refs/heads/tag-policy Commit: 104d1b897e33159f1019a1691bbca6b063332a5d Parents: 9d15be5 Author: Madhan Neethiraj Authored: Fri Jun 5 17:16:42 2015 -0700 Committer: Madhan Neethiraj Committed: Fri Jun 5 17:16:42 2015 -0700 ---------------------------------------------------------------------- .../org/apache/ranger/common/ServiceUtil.java | 2 +- .../ranger/patch/PatchMigration_J10002.java | 230 +++++++++++++------ 2 files changed, 159 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/104d1b89/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java index 2a84d6c..7c2bbdc 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java @@ -421,7 +421,7 @@ public class ServiceUtil { return ret; } - private static String toAccessType(int permType) { + public static String toAccessType(int permType) { String ret = null; for(Map.Entry e : mapAccessTypeToPermType.entrySet()) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/104d1b89/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java index 6df5b73..c33b39d 100644 --- a/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java +++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java @@ -18,33 +18,38 @@ package org.apache.ranger.patch; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.biz.ServiceDBStore; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.RangerCommonEnums; import org.apache.ranger.common.SearchCriteria; +import org.apache.ranger.common.ServiceUtil; import org.apache.ranger.common.StringUtil; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXAsset; import org.apache.ranger.entity.XXAuditMap; +import org.apache.ranger.entity.XXGroup; import org.apache.ranger.entity.XXPolicy; import org.apache.ranger.entity.XXPolicyConditionDef; import org.apache.ranger.entity.XXPortalUser; import org.apache.ranger.entity.XXResource; import org.apache.ranger.entity.XXServiceConfigDef; import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXUser; import org.apache.ranger.patch.BaseLoader; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.service.RangerPolicyService; @@ -52,7 +57,6 @@ import org.apache.ranger.service.XPermMapService; import org.apache.ranger.service.XPolicyService; import org.apache.ranger.util.CLIUtil; import org.apache.ranger.view.VXPermMap; -import org.apache.ranger.view.VXPermObj; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -207,10 +211,12 @@ public class PatchMigration_J10002 extends BaseLoader { RangerPolicy policy = new RangerPolicy(); policy = mapXResourceToPolicy(policy, xRes, service); - policy = svcDBStore.createPolicy(policy); + if(policy != null) { + policy = svcDBStore.createPolicy(policy); - policyCounter++; - logger.info("New policy created. policyName: " + policy.getName()); + policyCounter++; + logger.info("New policy created. policyName: " + policy.getName()); + } } svcDBStore.setPopulateExistingBaseFields(false); } catch (Exception e) { @@ -299,9 +305,8 @@ public class PatchMigration_J10002 extends BaseLoader { } private RangerPolicy mapXResourceToPolicy(RangerPolicy policy, XXResource xRes, RangerService service) { - String serviceName = service.getName(); - String serviceDef = service.getType(); + String serviceType = service.getType(); String name = xRes.getPolicyName(); String description = xRes.getDescription(); Boolean isAuditEnabled = true; @@ -309,6 +314,14 @@ public class PatchMigration_J10002 extends BaseLoader { Map resources = new HashMap(); List policyItems = new ArrayList(); + XXServiceDef svcDef = daoMgr.getXXServiceDef().findByName(serviceType); + + if(svcDef == null) { + logger.error(serviceType + ": service-def not found. Skipping policy '" + name + "'"); + + return null; + } + List auditMapList = daoMgr.getXXAuditMap().findByResourceId(xRes.getId()); if (stringUtil.isEmpty(auditMapList)) { isAuditEnabled = false; @@ -317,38 +330,29 @@ public class PatchMigration_J10002 extends BaseLoader { isEnabled = false; } - boolean tableExcludes = false; - boolean columnExcludes = false; - - if (xRes.getTableType() == AppConstants.POLICY_EXCLUSION) { - tableExcludes = true; - } - if (xRes.getColumnType() == AppConstants.POLICY_EXCLUSION) { - columnExcludes = true; - } - - if (serviceDef.equalsIgnoreCase("hdfs")) { - resources.put("path", new RangerPolicyResource(Arrays.asList(xRes.getName()), false, AppConstants - .getBooleanFor_BooleanValue(xRes.getIsRecursive()))); - - } else if (serviceDef.equalsIgnoreCase("hbase")) { - resources.put("table", new RangerPolicyResource(Arrays.asList(xRes.getTables()), tableExcludes, false)); - resources.put("column", new RangerPolicyResource(Arrays.asList(xRes.getColumns()), columnExcludes, false)); - resources.put("column-family", new RangerPolicyResource(Arrays.asList(xRes.getColumnFamilies()), false, false)); - - } else if (serviceDef.equalsIgnoreCase("hive")) { - resources.put("table", new RangerPolicyResource(Arrays.asList(xRes.getTables()), tableExcludes, false)); - resources.put("column", new RangerPolicyResource(Arrays.asList(xRes.getColumns()), columnExcludes, false)); - resources.put("database", new RangerPolicyResource(Arrays.asList(xRes.getDatabases()), false, false)); - resources.put("udf", new RangerPolicyResource(Arrays.asList(xRes.getUdfs()), false, false)); - } else if (serviceDef.equalsIgnoreCase("knox")) { - resources.put("topology", new RangerPolicyResource(Arrays.asList(xRes.getTopologies()), false, false)); - resources.put("service", new RangerPolicyResource(Arrays.asList(xRes.getServices()), false, false)); - } else if (serviceDef.equalsIgnoreCase("storm")) { - resources.put("topology", new RangerPolicyResource(Arrays.asList(xRes.getTopologies()), false, false)); + Boolean isPathRecursive = xRes.getIsRecursive() == RangerCommonEnums.BOOL_TRUE; + Boolean isTableExcludes = xRes.getTableType() == RangerCommonEnums.POLICY_EXCLUSION; + Boolean isColumnExcludes = xRes.getColumnType() == RangerCommonEnums.POLICY_EXCLUSION; + + if (StringUtils.equalsIgnoreCase(serviceType, "hdfs")) { + toRangerResourceList(xRes.getName(), "path", Boolean.FALSE, isPathRecursive, resources); + } else if (StringUtils.equalsIgnoreCase(serviceType, "hbase")) { + toRangerResourceList(xRes.getTables(), "table", isTableExcludes, Boolean.FALSE, resources); + toRangerResourceList(xRes.getColumnFamilies(), "column-family", Boolean.FALSE, Boolean.FALSE, resources); + toRangerResourceList(xRes.getColumns(), "column", isColumnExcludes, Boolean.FALSE, resources); + } else if (StringUtils.equalsIgnoreCase(serviceType, "hive")) { + toRangerResourceList(xRes.getDatabases(), "database", Boolean.FALSE, Boolean.FALSE, resources); + toRangerResourceList(xRes.getTables(), "table", isTableExcludes, Boolean.FALSE, resources); + toRangerResourceList(xRes.getColumns(), "column", isColumnExcludes, Boolean.FALSE, resources); + toRangerResourceList(xRes.getUdfs(), "udf", Boolean.FALSE, Boolean.FALSE, resources); + } else if (StringUtils.equalsIgnoreCase(serviceType, "knox")) { + toRangerResourceList(xRes.getTopologies(), "topology", Boolean.FALSE, Boolean.FALSE, resources); + toRangerResourceList(xRes.getServices(), "service", Boolean.FALSE, Boolean.FALSE, resources); + } else if (StringUtils.equalsIgnoreCase(serviceType, "storm")) { + toRangerResourceList(xRes.getTopologies(), "topology", Boolean.FALSE, Boolean.FALSE, resources); } - policyItems = getPolicyItemListForRes(xRes, serviceDef); + policyItems = getPolicyItemListForRes(xRes, svcDef); policy.setService(serviceName); policy.setName(name); @@ -376,60 +380,107 @@ public class PatchMigration_J10002 extends BaseLoader { return policy; } - private List getPolicyItemListForRes(XXResource xRes, String serviceDefName) { + private Map toRangerResourceList(String resourceString, String resourceType, Boolean isExcludes, Boolean isRecursive, Map resources) { + Map ret = resources == null ? new HashMap() : resources; + + if(StringUtils.isNotBlank(resourceString)) { + RangerPolicy.RangerPolicyResource resource = ret.get(resourceType); + + if(resource == null) { + resource = new RangerPolicy.RangerPolicyResource(); + resource.setIsExcludes(isExcludes); + resource.setIsRecursive(isRecursive); + + ret.put(resourceType, resource); + } + + for(String res : resourceString.split(",")) { + resource.getValues().add(res); + } + } + + return ret; + } + + private List getPolicyItemListForRes(XXResource xRes, XXServiceDef svcDef) { List policyItems = new ArrayList(); SearchCriteria sc = new SearchCriteria(); + sc.addParam("resourceId", xRes.getId()); List permMapList = xPermMapService.searchXPermMaps(sc).getVXPermMaps(); - List permObjList = xPolService.mapPermMapToPermObj(permMapList); - XXServiceDef svcDef = daoMgr.getXXServiceDef().findByName(serviceDefName); - if (svcDef == null) { - return new ArrayList(); - } + HashMap> sortedPermMap = new HashMap>(); - XXPolicyConditionDef policyCond = daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(svcDef.getId(), - "ip-range"); + // re-group the list with permGroup as the key + if (permMapList != null) { + for(VXPermMap permMap : permMapList) { + String permGrp = permMap.getPermGroup(); + List sortedList = sortedPermMap.get(permGrp); - for (VXPermObj permObj : permObjList) { + if(sortedList == null) { + sortedList = new ArrayList(); + sortedPermMap.put(permGrp, sortedList); + } - List permList = permObj.getPermList(); - if (permList == null) { - continue; + sortedList.add(permMap); } + } - RangerPolicyItem policyItem = new RangerPolicyItem(); - List accesses = new ArrayList(); - List conditions = new ArrayList(); + for (Entry> entry : sortedPermMap.entrySet()) { + List userList = new ArrayList(); + List groupList = new ArrayList(); + List accessList = new ArrayList(); + String ipAddress = null; - if (permObj.getPermList().contains("Admin")) { - policyItem.setDelegateAdmin(true); - } + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); - for (String perm : permList) { - RangerPolicyItemAccess access = new RangerPolicyItemAccess(); - access.setIsAllowed(true); - access.setType(perm); - accesses.add(access); - } - if (!stringUtil.isEmpty(permObj.getIpAddress()) && policyCond != null) { - RangerPolicyItemCondition condition = new RangerPolicyItemCondition(); - condition.setType("ip-range"); + for(VXPermMap permMap : entry.getValue()) { + if(permMap.getPermFor() == AppConstants.XA_PERM_FOR_USER) { + String userName = getUserName(permMap); + + if (! userList.contains(userName)) { + userList.add(userName); + } + } else if(permMap.getPermFor() == AppConstants.XA_PERM_FOR_GROUP) { + String groupName = getGroupName(permMap); - List ipRangeList = Arrays.asList(permObj.getIpAddress()); + if (! groupList.contains(groupName)) { + groupList.add(groupName); + } + } + + String accessType = ServiceUtil.toAccessType(permMap.getPermType()); + + if(StringUtils.equalsIgnoreCase(accessType, "Admin")) { + policyItem.setDelegateAdmin(Boolean.TRUE); + if ( svcDef.getId() == EmbeddedServiceDefsUtil.instance().getHBaseServiceDefId()) { + accessList.add(new RangerPolicyItemAccess(accessType)); + } + } else { + accessList.add(new RangerPolicyItemAccess(accessType)); + } - condition.setValues(ipRangeList); - conditions.add(condition); + ipAddress = permMap.getIpAddress(); } - policyItem.setUsers(permObj.getUserList()); - policyItem.setGroups(permObj.getGroupList()); - policyItem.setAccesses(accesses); - policyItem.setConditions(conditions); + policyItem.setUsers(userList); + policyItem.setGroups(groupList); + policyItem.setAccesses(accessList); + + if(ipAddress != null && !ipAddress.isEmpty()) { + XXPolicyConditionDef policyCond = daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(svcDef.getId(), "ip-range"); + + if(policyCond != null) { + RangerPolicy.RangerPolicyItemCondition ipCondition = new RangerPolicy.RangerPolicyItemCondition("ip-range", Collections.singletonList(ipAddress)); + + policyItem.getConditions().add(ipCondition); + } + } policyItems.add(policyItem); } + return policyItems; } @@ -477,4 +528,39 @@ public class PatchMigration_J10002 extends BaseLoader { } + private String getUserName(VXPermMap permMap) { + String userName = permMap.getUserName(); + + if(userName == null || userName.isEmpty()) { + Long userId = permMap.getUserId(); + + if(userId != null) { + XXUser xxUser = daoMgr.getXXUser().getById(userId); + + if(xxUser != null) { + userName = xxUser.getName(); + } + } + } + + return userName; + } + + private String getGroupName(VXPermMap permMap) { + String groupName = permMap.getGroupName(); + + if(groupName == null || groupName.isEmpty()) { + Long groupId = permMap.getGroupId(); + + if(groupId != null) { + XXGroup xxGroup = daoMgr.getXXGroup().getById(groupId); + + if(xxGroup != null) { + groupName = xxGroup.getName(); + } + } + } + + return groupName; + } } \ No newline at end of file