ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From v..@apache.org
Subject ranger git commit: RANGER-1357: tag objects are not removed when attempting a full sync with Atlas
Date Fri, 10 Feb 2017 23:41:22 GMT
Repository: ranger
Updated Branches:
  refs/heads/master fbc7f19db -> e4122919c


RANGER-1357: tag objects are not removed when attempting a full sync with Atlas

Signed-off-by: Velmurugan Periasamy <vel@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/e4122919
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/e4122919
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/e4122919

Branch: refs/heads/master
Commit: e4122919c46bd9fec38a76b27c7e0f5fb98a5f5b
Parents: fbc7f19
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Wed Feb 1 17:18:20 2017 -0800
Committer: Velmurugan Periasamy <vel@apache.org>
Committed: Fri Feb 10 18:41:09 2017 -0500

----------------------------------------------------------------------
 .../source/atlas/AtlasNotificationMapper.java   | 44 ++++++++++++--------
 .../source/atlasrest/AtlasRESTTagSource.java    |  8 ++--
 .../tagsync/source/atlasrest/AtlasRESTUtil.java | 28 ++++++-------
 3 files changed, 44 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index ec8508e..922317e 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -108,7 +108,7 @@ public class AtlasNotificationMapper {
 		return ret;
 	}
 
-	public static Map<String, ServiceTags> processEntitiesWithTraits(List<AtlasEntityWithTraits>
atlasEntities) {
+	public static Map<String, ServiceTags> processAtlasEntities(List<AtlasEntityWithTraits>
atlasEntities) {
 		Map<String, ServiceTags> ret = null;
 
 		try {
@@ -245,32 +245,40 @@ public class AtlasNotificationMapper {
 		RangerServiceResource  serviceResource = AtlasResourceMapperUtil.getRangerServiceResource(entity);
 
 		if (serviceResource != null) {
-			List<RangerTag>    tags    = getTags(entityWithTraits);
-			List<RangerTagDef> tagDefs = getTagDefs(entityWithTraits);
 
-			String serviceName = serviceResource.getServiceName();
+			List<RangerTag>     tags        = getTags(entityWithTraits);
+			List<RangerTagDef>  tagDefs     = getTagDefs(entityWithTraits);
+			String              serviceName = serviceResource.getServiceName();
 
 			ret = createOrGetServiceTags(serviceTagsMap, serviceName);
 
-			serviceResource.setId((long)ret.getServiceResources().size());
-			ret.getServiceResources().add(serviceResource);
+			if (serviceTagsMap == null || CollectionUtils.isNotEmpty(tags)) {
+
+				serviceResource.setId((long) ret.getServiceResources().size());
+				ret.getServiceResources().add(serviceResource);
 
-			List<Long> tagIds = new ArrayList<Long>();
+				List<Long> tagIds = new ArrayList<Long>();
 
-			if(CollectionUtils.isNotEmpty(tags)) {
-				for(RangerTag tag : tags) {
-					tag.setId((long)ret.getTags().size());
-					ret.getTags().put(tag.getId(), tag);
+				if (CollectionUtils.isNotEmpty(tags)) {
+					for (RangerTag tag : tags) {
+						tag.setId((long) ret.getTags().size());
+						ret.getTags().put(tag.getId(), tag);
 
-					tagIds.add(tag.getId());
+						tagIds.add(tag.getId());
+					}
 				}
-			}
-			ret.getResourceToTagIds().put(serviceResource.getId(), tagIds);
+				ret.getResourceToTagIds().put(serviceResource.getId(), tagIds);
 
-			if(CollectionUtils.isNotEmpty(tagDefs)) {
-				for(RangerTagDef tagDef : tagDefs) {
-					tagDef.setId((long)ret.getTagDefinitions().size());
-					ret.getTagDefinitions().put(tagDef.getId(), tagDef);
+				if (CollectionUtils.isNotEmpty(tagDefs)) {
+					for (RangerTagDef tagDef : tagDefs) {
+						tagDef.setId((long) ret.getTagDefinitions().size());
+						ret.getTagDefinitions().put(tagDef.getId(), tagDef);
+					}
+				}
+			} else {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Entity " + entityWithTraits + " does not have any tags associated with it
when full-sync is being done.");
+					LOG.debug("Will not add this entity to serviceTags, so that this entity, if exists,
 will be removed from ranger");
 				}
 			}
 		} else {

http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
index e0976cd..7da4ed4 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
@@ -194,16 +194,16 @@ public class AtlasRESTTagSource extends AbstractTagSource implements
Runnable {
 
 	public void synchUp() {
 
-		List<AtlasEntityWithTraits> atlasEntitiesWithTraits = atlasRESTUtil.getEntitiesWithTraits();
+		List<AtlasEntityWithTraits> atlasEntities = atlasRESTUtil.getAtlasEntities();
 
-		if (CollectionUtils.isNotEmpty(atlasEntitiesWithTraits)) {
+		if (CollectionUtils.isNotEmpty(atlasEntities)) {
 			if (LOG.isDebugEnabled()) {
-				for (AtlasEntityWithTraits element : atlasEntitiesWithTraits) {
+				for (AtlasEntityWithTraits element : atlasEntities) {
 					LOG.debug(element);
 				}
 			}
 
-			Map<String, ServiceTags> serviceTagsMap = AtlasNotificationMapper.processEntitiesWithTraits(atlasEntitiesWithTraits);
+			Map<String, ServiceTags> serviceTagsMap = AtlasNotificationMapper.processAtlasEntities(atlasEntities);
 
 			if (MapUtils.isNotEmpty(serviceTagsMap)) {
 				for (Map.Entry<String, ServiceTags> entry : serviceTagsMap.entrySet()) {

http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
index bf09cfd..167fe68 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
@@ -88,9 +88,9 @@ public class AtlasRESTUtil {
 		}
 	}
 
-	public List<AtlasEntityWithTraits> getEntitiesWithTraits() {
+	public List<AtlasEntityWithTraits> getAtlasEntities() {
 		if (LOG.isDebugEnabled()) {
-			LOG.debug("==> getEntriesWithTraits()");
+			LOG.debug("==> getAtlasEntities()");
 		}
 
 		List<AtlasEntityWithTraits> ret = new ArrayList<AtlasEntityWithTraits>();
@@ -129,9 +129,9 @@ public class AtlasRESTUtil {
 
 					Map<String, Object> traitsAttribute = getAttribute(definition, TRAITS_ATTRIBUTE,
Map.class);
 
-					if (MapUtils.isNotEmpty(traitsAttribute)) {
+					List<IStruct> allTraits = new LinkedList<>();
 
-						List<IStruct> allTraits = new LinkedList<>();
+					if (MapUtils.isNotEmpty(traitsAttribute)) {
 
 						for (Map.Entry<String, Object> entry : traitsAttribute.entrySet()) {
 
@@ -151,24 +151,24 @@ public class AtlasRESTUtil {
 							allTraits.add(trait1);
 							allTraits.addAll(superTypes);
 						}
+					}
 
-						IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition),
true);
+					IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition),
true);
 
-						if (entity != null) {
-							AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entity, allTraits);
-							ret.add(entityWithTraits);
-						} else {
-							if (LOG.isInfoEnabled()) {
-								LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid="
+ guid);
-							}
+					if (entity != null) {
+						AtlasEntityWithTraits atlasEntity = new AtlasEntityWithTraits(entity, allTraits);
+						ret.add(atlasEntity);
+					} else {
+						if (LOG.isInfoEnabled()) {
+							LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid="
+ guid);
 						}
-
 					}
 
 				}
+
 			}
 			if (LOG.isDebugEnabled()) {
-				LOG.debug("<== getEntriesWithTraits()");
+				LOG.debug("<== getAtlasEntities()");
 			}
 		}
 


Mime
View raw message