Return-Path: X-Original-To: apmail-ranger-commits-archive@www.apache.org Delivered-To: apmail-ranger-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8921518A68 for ; Wed, 28 Oct 2015 16:00:24 +0000 (UTC) Received: (qmail 60629 invoked by uid 500); 28 Oct 2015 16:00:11 -0000 Delivered-To: apmail-ranger-commits-archive@ranger.apache.org Received: (qmail 60600 invoked by uid 500); 28 Oct 2015 16:00:11 -0000 Mailing-List: contact commits-help@ranger.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ranger.incubator.apache.org Delivered-To: mailing list commits@ranger.incubator.apache.org Received: (qmail 60591 invoked by uid 99); 28 Oct 2015 16:00:11 -0000 Received: from Unknown (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Oct 2015 16:00:11 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id 5AAA4C0FD1 for ; Wed, 28 Oct 2015 16:00:11 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.771 X-Spam-Level: * X-Spam-Status: No, score=1.771 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-eu-west.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id d_7bPqOba6qC for ; Wed, 28 Oct 2015 16:00:02 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-eu-west.apache.org (ASF Mail Server at mx1-eu-west.apache.org) with SMTP id 7F07125B5B for ; Wed, 28 Oct 2015 15:59:43 +0000 (UTC) Received: (qmail 58435 invoked by uid 99); 28 Oct 2015 15:59:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Oct 2015 15:59:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AA492DFCCE; Wed, 28 Oct 2015 15:59:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@ranger.incubator.apache.org Date: Wed, 28 Oct 2015 16:00:11 -0000 Message-Id: <22959d3880a24d19bcabe25e1465edc8@git.apache.org> In-Reply-To: <5c3a8d58ae9641e6abda7e0069b42c82@git.apache.org> References: <5c3a8d58ae9641e6abda7e0069b42c82@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [35/50] [abbrv] incubator-ranger git commit: RANGER-660: updated Atlas notification handler to handle TRAIT_DELETED and ENTITY_UPDATED notifications RANGER-660: updated Atlas notification handler to handle TRAIT_DELETED and ENTITY_UPDATED notifications Signed-off-by: Madhan Neethiraj Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/5db14737 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/5db14737 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/5db14737 Branch: refs/heads/master Commit: 5db147378f004176638cddc626f6a0e9baa5efc1 Parents: 54270d5 Author: Abhay Kulkarni Authored: Thu Oct 1 15:08:21 2015 -0700 Committer: Madhan Neethiraj Committed: Thu Oct 1 19:07:41 2015 -0700 ---------------------------------------------------------------------- .../ranger/rest/ServiceTagsProcessor.java | 8 +- .../source/atlas/AtlasNotificationMapper.java | 153 ++++++++++--------- .../tagsync/source/atlas/TagAtlasSource.java | 2 +- 3 files changed, 88 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java index da99b72..b34313f 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java @@ -111,10 +111,10 @@ public class ServiceTagsProcessor { if(existing == null) { tagDefInStore = tagStore.createTagDef(tagDef); } else { - tagDef.setId(existing.getId()); - tagDef.setGuid(existing.getGuid()); - - tagDefInStore = tagStore.updateTagDef(tagDef); + if (LOG.isDebugEnabled()) { + LOG.debug("tagDef for name:" + tagDef.getName() + " exists, will not update it"); + } + tagDefInStore = existing; } if(tagDefsInStore != null) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index dd5cd44..13d67b3 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -94,19 +94,20 @@ class AtlasNotificationMapper { String opName = entityNotification.getOperationType().name(); switch (opType) { case ENTITY_CREATED: { - ret = getServiceTags(entity, opType); + LOG.info("ENTITY_CREATED notification is not handled, as Ranger will get necessary information from any subsequent TRAIT_ADDED notification"); break; } case ENTITY_UPDATED: { - ret = handleEntityUpdate(entity); - break; - } - case TRAIT_ADDED: { - ret = getServiceTags(entity, opType); + ret = getServiceTags(entity); + if (MapUtils.isEmpty(ret.getTags())) { + LOG.info("No traits associated with this entity update notification. Ignoring it altogether"); + ret = null; + } break; } + case TRAIT_ADDED: case TRAIT_DELETED: { - ret = handleTraitDelete(entity); + ret = getServiceTags(entity); break; } default: @@ -116,31 +117,36 @@ class AtlasNotificationMapper { return ret; } - static private ServiceTags getServiceTags(Entity entity, EntityNotification.OperationType opType) throws Exception { + static private ServiceTags getServiceTags(Entity entity) throws Exception { ServiceTags ret = null; List serviceResources = new ArrayList(); - RangerServiceResource serviceResource = getServiceResource(entity, opType); + RangerServiceResource serviceResource = getServiceResource(entity); serviceResources.add(serviceResource); Map tags = getTags(entity); - Map tagDefs = getTagDefs(tags, EntityNotification.OperationType.ENTITY_CREATED); + Map tagDefs = getTagDefs(tags); Map> resourceIdToTagIds = null; + resourceIdToTagIds = new HashMap>(); + List tagList = new ArrayList(); + + if (MapUtils.isNotEmpty(tags)) { resourceIdToTagIds = new HashMap>(); - List tagList = new ArrayList(); for (Map.Entry entry : tags.entrySet()) { tagList.add(entry.getKey()); } - resourceIdToTagIds.put(1L, tagList); } + resourceIdToTagIds.put(1L, tagList); + + ret = new ServiceTags(); ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); @@ -154,68 +160,67 @@ class AtlasNotificationMapper { } - static private RangerServiceResource getServiceResource(Entity entity, EntityNotification.OperationType opType) throws Exception { + static private RangerServiceResource getServiceResource(Entity entity) throws Exception { RangerServiceResource ret = null; Map elements = null; String serviceName = null; - if (opType == EntityNotification.OperationType.ENTITY_CREATED) { - elements = new HashMap(); + elements = new HashMap(); - String[] components = getQualifiedNameComponents(entity); - // components should contain qualifiedName, instanceName, dbName, tableName, columnName in that order + String[] components = getQualifiedNameComponents(entity); + // components should contain qualifiedName, instanceName, dbName, tableName, columnName in that order - String entityTypeName = entity.getTypeName(); + String entityTypeName = entity.getTypeName(); - String instanceName, dbName, tableName, columnName; + String instanceName, dbName, tableName, columnName; - if (components.length > 1) { - instanceName = components[1]; - serviceName = getServiceName(instanceName, entityTypeName); - } - - if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) { - if (components.length > 2) { - dbName = components[2]; - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); + if (components.length > 1) { + instanceName = components[1]; + serviceName = getServiceName(instanceName, entityTypeName); + } - } else { - LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]); - } - } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) { - if (components.length > 3) { - dbName = components[2]; - tableName = components[3]; - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); - elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); - } else { - LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]); - } - } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) { - if (components.length > 4) { - dbName = components[2]; - tableName = components[3]; - columnName = components[4]; - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); - elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); - RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName); - elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource); - } else { - LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]); - } + if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) { + if (components.length > 2) { + dbName = components[2]; + RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); + elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); + } else { + LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]); + } + } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) { + if (components.length > 3) { + dbName = components[2]; + tableName = components[3]; + RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); + elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); + RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); + elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); + } else { + LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]); } + } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) { + if (components.length > 4) { + dbName = components[2]; + tableName = components[3]; + columnName = components[4]; + RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); + elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); + RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); + elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); + RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName); + elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource); + } else { + LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]); + } + } + ret = new RangerServiceResource(); ret.setGuid(entity.getId().getGuid()); ret.setId(1L); @@ -255,7 +260,6 @@ class AtlasNotificationMapper { RangerTag tag = new RangerTag(); - tag.setGuid(entity.getId().getGuid() + "-" + traitName); tag.setType(traitName); tag.setAttributes(tagAttrValues); @@ -266,19 +270,17 @@ class AtlasNotificationMapper { return ret; } - static private Map getTagDefs(Map tags, EntityNotification.OperationType opType) { + static private Map getTagDefs(Map tags) { Map ret = null; - if (opType == EntityNotification.OperationType.ENTITY_CREATED || opType == EntityNotification.OperationType.TRAIT_ADDED) { - if (MapUtils.isNotEmpty(tags)) { - ret = new HashMap(); - for (Map.Entry entry : tags.entrySet()) { - RangerTagDef tagDef = new RangerTagDef(); - tagDef.setName(entry.getValue().getType()); - tagDef.setId(entry.getKey()); - ret.put(entry.getKey(), tagDef); - } + if (MapUtils.isNotEmpty(tags)) { + ret = new HashMap(); + for (Map.Entry entry : tags.entrySet()) { + RangerTagDef tagDef = new RangerTagDef(); + tagDef.setName(entry.getValue().getType()); + tagDef.setId(entry.getKey()); + ret.put(entry.getKey(), tagDef); } } @@ -289,10 +291,21 @@ class AtlasNotificationMapper { String ret[] = new String[5]; if (StringUtils.equals(entity.getTypeName(), ENTITY_TYPE_HIVE_DB)) { - ret[1] = getAttribute(entity.getValues(), "clusterName", String.class); - ret[2] = getAttribute(entity.getValues(), "name", String.class); + + String clusterName = getAttribute(entity.getValues(), "clusterName", String.class); + String name = getAttribute(entity.getValues(), "name", String.class); + + ret[1] = clusterName; + ret[2] = name; ret[3] = null; ret[0] = ret[1] + "." + ret[2]; + + if (LOG.isDebugEnabled()) { + LOG.debug("----- Entity-Id:" + entity.getId().getGuid()); + LOG.debug("----- Entity-Type-Name:" + entity.getTypeName()); + LOG.debug("----- Entity-Cluster-Name:" + clusterName); + LOG.debug("----- Entity-Name:" + name); + } } else { String qualifiedName = getAttribute(entity.getValues(), ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java index 243aee5..0c0dee9 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java @@ -151,7 +151,7 @@ public class TagAtlasSource implements TagSource { printNotification(notification); ServiceTags serviceTags = AtlasNotificationMapper.processEntityNotification(notification, properties); if (serviceTags == null) { - LOG.error("Failed to map Atlas notification to ServiceTags structure"); + LOG.info("Did not create ServiceTags structure for notification type:" + notification.getOperationType().name()); } else { if (LOG.isDebugEnabled()) { String serviceTagsJSON = new Gson().toJson(serviceTags);