ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [35/50] [abbrv] incubator-ranger git commit: RANGER-660: updated Atlas notification handler to handle TRAIT_DELETED and ENTITY_UPDATED notifications
Date Wed, 28 Oct 2015 16:00:11 GMT
RANGER-660: updated Atlas notification handler to handle TRAIT_DELETED and ENTITY_UPDATED notifications

Signed-off-by: Madhan Neethiraj <madhan@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/5db14737
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/5db14737
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/5db14737

Branch: refs/heads/master
Commit: 5db147378f004176638cddc626f6a0e9baa5efc1
Parents: 54270d5
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Thu Oct 1 15:08:21 2015 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Thu Oct 1 19:07:41 2015 -0700

----------------------------------------------------------------------
 .../ranger/rest/ServiceTagsProcessor.java       |   8 +-
 .../source/atlas/AtlasNotificationMapper.java   | 153 ++++++++++---------
 .../tagsync/source/atlas/TagAtlasSource.java    |   2 +-
 3 files changed, 88 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
index da99b72..b34313f 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
@@ -111,10 +111,10 @@ public class ServiceTagsProcessor {
 					if(existing == null) {
 						tagDefInStore = tagStore.createTagDef(tagDef);
 					} else {
-						tagDef.setId(existing.getId());
-						tagDef.setGuid(existing.getGuid());
-
-						tagDefInStore = tagStore.updateTagDef(tagDef);
+						if (LOG.isDebugEnabled()) {
+							LOG.debug("tagDef for name:" + tagDef.getName() + " exists, will not update it");
+						}
+						tagDefInStore = existing;
 					}
 
 					if(tagDefsInStore != null) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index dd5cd44..13d67b3 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -94,19 +94,20 @@ class AtlasNotificationMapper {
 		String opName = entityNotification.getOperationType().name();
 		switch (opType) {
 			case ENTITY_CREATED: {
-				ret = getServiceTags(entity, opType);
+				LOG.info("ENTITY_CREATED notification is not handled, as Ranger will get necessary information
from any subsequent TRAIT_ADDED notification");
 				break;
 			}
 			case ENTITY_UPDATED: {
-				ret = handleEntityUpdate(entity);
-				break;
-			}
-			case TRAIT_ADDED: {
-				ret = getServiceTags(entity, opType);
+				ret = getServiceTags(entity);
+				if (MapUtils.isEmpty(ret.getTags())) {
+					LOG.info("No traits associated with this entity update notification. Ignoring it altogether");
+					ret = null;
+				}
 				break;
 			}
+			case TRAIT_ADDED:
 			case TRAIT_DELETED: {
-				ret = handleTraitDelete(entity);
+				ret = getServiceTags(entity);
 				break;
 			}
 			default:
@@ -116,31 +117,36 @@ class AtlasNotificationMapper {
 		return ret;
 	}
 
-	static private ServiceTags getServiceTags(Entity entity, EntityNotification.OperationType
opType) throws Exception {
+	static private ServiceTags getServiceTags(Entity entity) throws Exception {
 		ServiceTags ret = null;
 
 
 		List<RangerServiceResource> serviceResources = new ArrayList<RangerServiceResource>();
 
-		RangerServiceResource serviceResource = getServiceResource(entity, opType);
+		RangerServiceResource serviceResource = getServiceResource(entity);
 		serviceResources.add(serviceResource);
 
 		Map<Long, RangerTag> tags = getTags(entity);
 
-		Map<Long, RangerTagDef> tagDefs = getTagDefs(tags, EntityNotification.OperationType.ENTITY_CREATED);
+		Map<Long, RangerTagDef> tagDefs = getTagDefs(tags);
 
 		Map<Long, List<Long>> resourceIdToTagIds = null;
 
+		resourceIdToTagIds = new HashMap<Long, List<Long>>();
+		List<Long> tagList = new ArrayList<Long>();
+
+
 		if (MapUtils.isNotEmpty(tags)) {
 			resourceIdToTagIds = new HashMap<Long, List<Long>>();
 
-			List<Long> tagList = new ArrayList<Long>();
 			for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) {
 				tagList.add(entry.getKey());
 			}
-			resourceIdToTagIds.put(1L, tagList);
 		}
 
+		resourceIdToTagIds.put(1L, tagList);
+
+
 		ret = new ServiceTags();
 
 		ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
@@ -154,68 +160,67 @@ class AtlasNotificationMapper {
 	}
 
 
-	static private RangerServiceResource getServiceResource(Entity entity, EntityNotification.OperationType
opType) throws Exception {
+	static private RangerServiceResource getServiceResource(Entity entity) throws Exception
{
 
 		RangerServiceResource ret = null;
 
 		Map<String, RangerPolicy.RangerPolicyResource> elements = null;
 		String serviceName = null;
 
-		if (opType == EntityNotification.OperationType.ENTITY_CREATED) {
 
-			elements = new HashMap<String, RangerPolicy.RangerPolicyResource>();
+		elements = new HashMap<String, RangerPolicy.RangerPolicyResource>();
 
-			String[] components = getQualifiedNameComponents(entity);
-			// components should contain qualifiedName, instanceName, dbName, tableName, columnName
in that order
+		String[] components = getQualifiedNameComponents(entity);
+		// components should contain qualifiedName, instanceName, dbName, tableName, columnName
in that order
 
 
-			String entityTypeName = entity.getTypeName();
+		String entityTypeName = entity.getTypeName();
 
-			String instanceName, dbName, tableName, columnName;
+		String instanceName, dbName, tableName, columnName;
 
-			if (components.length > 1) {
-				instanceName = components[1];
-				serviceName = getServiceName(instanceName, entityTypeName);
-			}
-
-			if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) {
-				if (components.length > 2) {
-					dbName = components[2];
-					RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
-					elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
+		if (components.length > 1) {
+			instanceName = components[1];
+			serviceName = getServiceName(instanceName, entityTypeName);
+		}
 
-				} else {
-					LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]);
-				}
-			} else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) {
-				if (components.length > 3) {
-					dbName = components[2];
-					tableName = components[3];
-					RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
-					elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
-					RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
-					elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
-				} else {
-					LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]);
-				}
-			} else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) {
-				if (components.length > 4) {
-					dbName = components[2];
-					tableName = components[3];
-					columnName = components[4];
-					RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
-					elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
-					RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
-					elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
-					RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName);
-					elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource);
-				} else {
-					LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]);
-				}
+		if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) {
+			if (components.length > 2) {
+				dbName = components[2];
+				RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
+				elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
 
+			} else {
+				LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]);
+			}
+		} else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) {
+			if (components.length > 3) {
+				dbName = components[2];
+				tableName = components[3];
+				RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
+				elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
+				RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
+				elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
+			} else {
+				LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]);
 			}
+		} else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) {
+			if (components.length > 4) {
+				dbName = components[2];
+				tableName = components[3];
+				columnName = components[4];
+				RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
+				elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
+				RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
+				elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
+				RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName);
+				elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource);
+			} else {
+				LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]);
+			}
+
 		}
 
+
 		ret = new RangerServiceResource();
 		ret.setGuid(entity.getId().getGuid());
 		ret.setId(1L);
@@ -255,7 +260,6 @@ class AtlasNotificationMapper {
 
 				RangerTag tag = new RangerTag();
 
-				tag.setGuid(entity.getId().getGuid() + "-" + traitName);
 				tag.setType(traitName);
 				tag.setAttributes(tagAttrValues);
 
@@ -266,19 +270,17 @@ class AtlasNotificationMapper {
 		return ret;
 	}
 
-	static private Map<Long, RangerTagDef> getTagDefs(Map<Long, RangerTag> tags,
EntityNotification.OperationType opType) {
+	static private Map<Long, RangerTagDef> getTagDefs(Map<Long, RangerTag> tags)
{
 
 		Map<Long, RangerTagDef> ret = null;
 
-		if (opType == EntityNotification.OperationType.ENTITY_CREATED || opType == EntityNotification.OperationType.TRAIT_ADDED)
{
-			if (MapUtils.isNotEmpty(tags)) {
-				ret = new HashMap<Long, RangerTagDef>();
-				for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) {
-					RangerTagDef tagDef = new RangerTagDef();
-					tagDef.setName(entry.getValue().getType());
-					tagDef.setId(entry.getKey());
-					ret.put(entry.getKey(), tagDef);
-				}
+		if (MapUtils.isNotEmpty(tags)) {
+			ret = new HashMap<Long, RangerTagDef>();
+			for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) {
+				RangerTagDef tagDef = new RangerTagDef();
+				tagDef.setName(entry.getValue().getType());
+				tagDef.setId(entry.getKey());
+				ret.put(entry.getKey(), tagDef);
 			}
 		}
 
@@ -289,10 +291,21 @@ class AtlasNotificationMapper {
 		String ret[] = new String[5];
 
 		if (StringUtils.equals(entity.getTypeName(), ENTITY_TYPE_HIVE_DB)) {
-			ret[1] = getAttribute(entity.getValues(), "clusterName", String.class);
-			ret[2] = getAttribute(entity.getValues(), "name", String.class);
+
+			String clusterName = getAttribute(entity.getValues(), "clusterName", String.class);
+			String name = getAttribute(entity.getValues(), "name", String.class);
+
+			ret[1] = clusterName;
+			ret[2] = name;
 			ret[3] = null;
 			ret[0] = ret[1] + "." + ret[2];
+
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("----- Entity-Id:" + entity.getId().getGuid());
+				LOG.debug("----- Entity-Type-Name:" + entity.getTypeName());
+				LOG.debug("----- Entity-Cluster-Name:" + clusterName);
+				LOG.debug("----- Entity-Name:" + name);
+			}
 		} else {
 			String qualifiedName = getAttribute(entity.getValues(), ENTITY_ATTRIBUTE_QUALIFIED_NAME,
String.class);
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
index 243aee5..0c0dee9 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
@@ -151,7 +151,7 @@ public class TagAtlasSource implements TagSource {
 						printNotification(notification);
 						ServiceTags serviceTags = AtlasNotificationMapper.processEntityNotification(notification,
properties);
 						if (serviceTags == null) {
-							LOG.error("Failed to map Atlas notification to ServiceTags structure");
+							LOG.info("Did not create ServiceTags structure for notification type:" + notification.getOperationType().name());
 						} else {
 							if (LOG.isDebugEnabled()) {
 								String serviceTagsJSON = new Gson().toJson(serviceTags);


Mime
View raw message