Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C8219200C26 for ; Sat, 11 Feb 2017 00:41:23 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id C69A9160B69; Fri, 10 Feb 2017 23:41:23 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EB519160B5C for ; Sat, 11 Feb 2017 00:41:22 +0100 (CET) Received: (qmail 19234 invoked by uid 500); 10 Feb 2017 23:41:22 -0000 Mailing-List: contact commits-help@ranger.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ranger.apache.org Delivered-To: mailing list commits@ranger.apache.org Received: (qmail 19225 invoked by uid 99); 10 Feb 2017 23:41:22 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Feb 2017 23:41:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 17D09DFBDB; Fri, 10 Feb 2017 23:41:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vel@apache.org To: commits@ranger.apache.org Message-Id: <9998618daafc4edba7fe792d2b683ad9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ranger git commit: RANGER-1357: tag objects are not removed when attempting a full sync with Atlas Date: Fri, 10 Feb 2017 23:41:22 +0000 (UTC) archived-at: Fri, 10 Feb 2017 23:41:24 -0000 Repository: ranger Updated Branches: refs/heads/master fbc7f19db -> e4122919c RANGER-1357: tag objects are not removed when attempting a full sync with Atlas Signed-off-by: Velmurugan Periasamy Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/e4122919 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/e4122919 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/e4122919 Branch: refs/heads/master Commit: e4122919c46bd9fec38a76b27c7e0f5fb98a5f5b Parents: fbc7f19 Author: Abhay Kulkarni Authored: Wed Feb 1 17:18:20 2017 -0800 Committer: Velmurugan Periasamy Committed: Fri Feb 10 18:41:09 2017 -0500 ---------------------------------------------------------------------- .../source/atlas/AtlasNotificationMapper.java | 44 ++++++++++++-------- .../source/atlasrest/AtlasRESTTagSource.java | 8 ++-- .../tagsync/source/atlasrest/AtlasRESTUtil.java | 28 ++++++------- 3 files changed, 44 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index ec8508e..922317e 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -108,7 +108,7 @@ public class AtlasNotificationMapper { return ret; } - public static Map processEntitiesWithTraits(List atlasEntities) { + public static Map processAtlasEntities(List atlasEntities) { Map ret = null; try { @@ -245,32 +245,40 @@ public class AtlasNotificationMapper { RangerServiceResource serviceResource = AtlasResourceMapperUtil.getRangerServiceResource(entity); if (serviceResource != null) { - List tags = getTags(entityWithTraits); - List tagDefs = getTagDefs(entityWithTraits); - String serviceName = serviceResource.getServiceName(); + List tags = getTags(entityWithTraits); + List tagDefs = getTagDefs(entityWithTraits); + String serviceName = serviceResource.getServiceName(); ret = createOrGetServiceTags(serviceTagsMap, serviceName); - serviceResource.setId((long)ret.getServiceResources().size()); - ret.getServiceResources().add(serviceResource); + if (serviceTagsMap == null || CollectionUtils.isNotEmpty(tags)) { + + serviceResource.setId((long) ret.getServiceResources().size()); + ret.getServiceResources().add(serviceResource); - List tagIds = new ArrayList(); + List tagIds = new ArrayList(); - if(CollectionUtils.isNotEmpty(tags)) { - for(RangerTag tag : tags) { - tag.setId((long)ret.getTags().size()); - ret.getTags().put(tag.getId(), tag); + if (CollectionUtils.isNotEmpty(tags)) { + for (RangerTag tag : tags) { + tag.setId((long) ret.getTags().size()); + ret.getTags().put(tag.getId(), tag); - tagIds.add(tag.getId()); + tagIds.add(tag.getId()); + } } - } - ret.getResourceToTagIds().put(serviceResource.getId(), tagIds); + ret.getResourceToTagIds().put(serviceResource.getId(), tagIds); - if(CollectionUtils.isNotEmpty(tagDefs)) { - for(RangerTagDef tagDef : tagDefs) { - tagDef.setId((long)ret.getTagDefinitions().size()); - ret.getTagDefinitions().put(tagDef.getId(), tagDef); + if (CollectionUtils.isNotEmpty(tagDefs)) { + for (RangerTagDef tagDef : tagDefs) { + tagDef.setId((long) ret.getTagDefinitions().size()); + ret.getTagDefinitions().put(tagDef.getId(), tagDef); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Entity " + entityWithTraits + " does not have any tags associated with it when full-sync is being done."); + LOG.debug("Will not add this entity to serviceTags, so that this entity, if exists, will be removed from ranger"); } } } else { http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index e0976cd..7da4ed4 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -194,16 +194,16 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { public void synchUp() { - List atlasEntitiesWithTraits = atlasRESTUtil.getEntitiesWithTraits(); + List atlasEntities = atlasRESTUtil.getAtlasEntities(); - if (CollectionUtils.isNotEmpty(atlasEntitiesWithTraits)) { + if (CollectionUtils.isNotEmpty(atlasEntities)) { if (LOG.isDebugEnabled()) { - for (AtlasEntityWithTraits element : atlasEntitiesWithTraits) { + for (AtlasEntityWithTraits element : atlasEntities) { LOG.debug(element); } } - Map serviceTagsMap = AtlasNotificationMapper.processEntitiesWithTraits(atlasEntitiesWithTraits); + Map serviceTagsMap = AtlasNotificationMapper.processAtlasEntities(atlasEntities); if (MapUtils.isNotEmpty(serviceTagsMap)) { for (Map.Entry entry : serviceTagsMap.entrySet()) { http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java index bf09cfd..167fe68 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java @@ -88,9 +88,9 @@ public class AtlasRESTUtil { } } - public List getEntitiesWithTraits() { + public List getAtlasEntities() { if (LOG.isDebugEnabled()) { - LOG.debug("==> getEntriesWithTraits()"); + LOG.debug("==> getAtlasEntities()"); } List ret = new ArrayList(); @@ -129,9 +129,9 @@ public class AtlasRESTUtil { Map traitsAttribute = getAttribute(definition, TRAITS_ATTRIBUTE, Map.class); - if (MapUtils.isNotEmpty(traitsAttribute)) { + List allTraits = new LinkedList<>(); - List allTraits = new LinkedList<>(); + if (MapUtils.isNotEmpty(traitsAttribute)) { for (Map.Entry entry : traitsAttribute.entrySet()) { @@ -151,24 +151,24 @@ public class AtlasRESTUtil { allTraits.add(trait1); allTraits.addAll(superTypes); } + } - IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition), true); + IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition), true); - if (entity != null) { - AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entity, allTraits); - ret.add(entityWithTraits); - } else { - if (LOG.isInfoEnabled()) { - LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid=" + guid); - } + if (entity != null) { + AtlasEntityWithTraits atlasEntity = new AtlasEntityWithTraits(entity, allTraits); + ret.add(atlasEntity); + } else { + if (LOG.isInfoEnabled()) { + LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid=" + guid); } - } } + } if (LOG.isDebugEnabled()) { - LOG.debug("<== getEntriesWithTraits()"); + LOG.debug("<== getAtlasEntities()"); } }