Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 19E06200C1D for ; Thu, 16 Feb 2017 09:35:04 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 1868A160B61; Thu, 16 Feb 2017 08:35:04 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D1389160B57 for ; Thu, 16 Feb 2017 09:35:01 +0100 (CET) Received: (qmail 56009 invoked by uid 500); 16 Feb 2017 08:35:01 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 56000 invoked by uid 99); 16 Feb 2017 08:35:01 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 16 Feb 2017 08:35:01 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 7CBA4180695 for ; Thu, 16 Feb 2017 08:35:00 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.218 X-Spam-Level: X-Spam-Status: No, score=-6.218 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id VpgLVOgTp8Xz for ; Thu, 16 Feb 2017 08:34:51 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 4F91C5F19B for ; Thu, 16 Feb 2017 08:34:49 +0000 (UTC) Received: (qmail 55102 invoked by uid 99); 16 Feb 2017 08:34:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 16 Feb 2017 08:34:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6B442DFC69; Thu, 16 Feb 2017 08:34:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@atlas.incubator.apache.org Date: Thu, 16 Feb 2017 08:34:49 -0000 Message-Id: <350bea51f6504dc2be911f5f11d3f72d@git.apache.org> In-Reply-To: <1e9735518d18461a8a77bda520d0b483@git.apache.org> References: <1e9735518d18461a8a77bda520d0b483@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] incubator-atlas git commit: ATLAS-1554 : v2 EntityREST implementation for entity partial update archived-at: Thu, 16 Feb 2017 08:35:04 -0000 ATLAS-1554 : v2 EntityREST implementation for entity partial update Signed-off-by: Madhan Neethiraj Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/68c55925 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/68c55925 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/68c55925 Branch: refs/heads/master Commit: 68c559254336b8c6d7dbd6537b064d4d31d0529d Parents: c5ea4f0 Author: Sarath Subramanian Authored: Tue Feb 14 17:54:04 2017 -0800 Committer: Madhan Neethiraj Committed: Thu Feb 16 00:17:42 2017 -0800 ---------------------------------------------------------------------- .../model/instance/EntityMutationResponse.java | 65 ++-- .../org/apache/atlas/type/AtlasArrayType.java | 139 +++++++++ .../atlas/type/AtlasClassificationType.java | 69 +++++ .../org/apache/atlas/type/AtlasEntityType.java | 106 +++++-- .../org/apache/atlas/type/AtlasMapType.java | 89 ++++++ .../org/apache/atlas/type/AtlasStructType.java | 178 ++++++++++- .../java/org/apache/atlas/type/AtlasType.java | 8 + .../test/java/org/apache/atlas/TestUtilsV2.java | 1 + .../store/graph/AtlasEntityStore.java | 3 +- .../store/graph/EntityGraphDiscovery.java | 4 + .../graph/v1/AtlasEntityGraphDiscoveryV1.java | 74 +++-- .../store/graph/v1/AtlasEntityStoreV1.java | 59 ++-- .../store/graph/v1/EntityGraphMapper.java | 10 +- .../graph/v1/AtlasDeleteHandlerV1Test.java | 32 +- .../store/graph/v1/AtlasEntityStoreV1Test.java | 299 +++++++++++++++---- .../org/apache/atlas/web/rest/EntityREST.java | 24 +- .../atlas/web/adapters/TestEntityREST.java | 36 ++- 17 files changed, 997 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java index 45b12e3..5e8ce35 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java @@ -32,6 +32,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.model.instance.EntityMutations.EntityOperation; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnore; import org.codehaus.jackson.annotate.JsonIgnoreProperties; @@ -44,21 +45,21 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; @XmlAccessorType(XmlAccessType.PROPERTY) public class EntityMutationResponse { - Map> mutatedEntities; - Map guidAssignments; + Map> mutatedEntities; + Map guidAssignments; public EntityMutationResponse() { } - public EntityMutationResponse(final Map> mutatedEntities) { + public EntityMutationResponse(final Map> mutatedEntities) { this.mutatedEntities = mutatedEntities; } - public Map> getMutatedEntities() { + public Map> getMutatedEntities() { return mutatedEntities; } - public void setMutatedEntities(final Map> mutatedEntities) { + public void setMutatedEntities(final Map> mutatedEntities) { this.mutatedEntities = mutatedEntities; } @@ -72,7 +73,7 @@ public class EntityMutationResponse { @JsonIgnore - public List getEntitiesByOperation(EntityMutations.EntityOperation op) { + public List getEntitiesByOperation(EntityOperation op) { if ( mutatedEntities != null) { return mutatedEntities.get(op); } @@ -82,7 +83,7 @@ public class EntityMutationResponse { @JsonIgnore public List getCreatedEntities() { if ( mutatedEntities != null) { - return mutatedEntities.get(EntityMutations.EntityOperation.CREATE); + return mutatedEntities.get(EntityOperation.CREATE); } return null; } @@ -90,7 +91,14 @@ public class EntityMutationResponse { @JsonIgnore public List getUpdatedEntities() { if ( mutatedEntities != null) { - return mutatedEntities.get(EntityMutations.EntityOperation.UPDATE); + return mutatedEntities.get(EntityOperation.UPDATE); + } + return null; + } + + public List getPartialUpdatedEntities() { + if ( mutatedEntities != null) { + return mutatedEntities.get(EntityOperation.PARTIAL_UPDATE); } return null; } @@ -98,14 +106,14 @@ public class EntityMutationResponse { @JsonIgnore public List getDeletedEntities() { if ( mutatedEntities != null) { - return mutatedEntities.get(EntityMutations.EntityOperation.DELETE); + return mutatedEntities.get(EntityOperation.DELETE); } return null; } @JsonIgnore public AtlasEntityHeader getFirstEntityCreated() { - final List entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.CREATE); + final List entitiesByOperation = getEntitiesByOperation(EntityOperation.CREATE); if ( entitiesByOperation != null && entitiesByOperation.size() > 0) { return entitiesByOperation.get(0); } @@ -115,7 +123,18 @@ public class EntityMutationResponse { @JsonIgnore public AtlasEntityHeader getFirstEntityUpdated() { - final List entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE); + final List entitiesByOperation = getEntitiesByOperation(EntityOperation.UPDATE); + + if ( entitiesByOperation != null && entitiesByOperation.size() > 0) { + return entitiesByOperation.get(0); + } + + return null; + } + + @JsonIgnore + public AtlasEntityHeader getFirstEntityPartialUpdated() { + final List entitiesByOperation = getEntitiesByOperation(EntityOperation.PARTIAL_UPDATE); if ( entitiesByOperation != null && entitiesByOperation.size() > 0) { return entitiesByOperation.get(0); } @@ -125,45 +144,51 @@ public class EntityMutationResponse { @JsonIgnore public AtlasEntityHeader getFirstCreatedEntityByTypeName(String typeName) { - return getFirstEntityByType(getEntitiesByOperation(EntityMutations.EntityOperation.CREATE), typeName); + return getFirstEntityByType(getEntitiesByOperation(EntityOperation.CREATE), typeName); } @JsonIgnore public AtlasEntityHeader getFirstDeletedEntityByTypeName(String typeName) { - return getFirstEntityByType(getEntitiesByOperation(EntityMutations.EntityOperation.DELETE), typeName); + return getFirstEntityByType(getEntitiesByOperation(EntityOperation.DELETE), typeName); } @JsonIgnore public List getCreatedEntitiesByTypeName(String typeName) { - return getEntitiesByType(getEntitiesByOperation(EntityMutations.EntityOperation.CREATE), typeName); + return getEntitiesByType(getEntitiesByOperation(EntityOperation.CREATE), typeName); + } + + @JsonIgnore + public List getPartialUpdatedEntitiesByTypeName(String typeName) { + return getEntitiesByType(getEntitiesByOperation(EntityOperation.PARTIAL_UPDATE), typeName); } @JsonIgnore public AtlasEntityHeader getCreatedEntityByTypeNameAndAttribute(String typeName, String attrName, String attrVal) { - return getEntityByTypeAndUniqueAttribute(getEntitiesByOperation(EntityMutations.EntityOperation.CREATE), typeName, attrName, attrVal); + return getEntityByTypeAndUniqueAttribute(getEntitiesByOperation(EntityOperation.CREATE), typeName, attrName, attrVal); } @JsonIgnore + public AtlasEntityHeader getUpdatedEntityByTypeNameAndAttribute(String typeName, String attrName, String attrVal) { - return getEntityByTypeAndUniqueAttribute(getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE), typeName, attrName, attrVal); + return getEntityByTypeAndUniqueAttribute(getEntitiesByOperation(EntityOperation.UPDATE), typeName, attrName, attrVal); } @JsonIgnore public List getUpdatedEntitiesByTypeName(String typeName) { - return getEntitiesByType(getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE), typeName); + return getEntitiesByType(getEntitiesByOperation(EntityOperation.UPDATE), typeName); } @JsonIgnore public List getDeletedEntitiesByTypeName(String typeName) { - return getEntitiesByType(getEntitiesByOperation(EntityMutations.EntityOperation.DELETE), typeName); + return getEntitiesByType(getEntitiesByOperation(EntityOperation.DELETE), typeName); } @JsonIgnore public AtlasEntityHeader getFirstUpdatedEntityByTypeName(String typeName) { - return getFirstEntityByType(getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE), typeName); + return getFirstEntityByType(getEntitiesByOperation(EntityOperation.UPDATE), typeName); } - public void addEntity(EntityMutations.EntityOperation op, AtlasEntityHeader header) { + public void addEntity(EntityOperation op, AtlasEntityHeader header) { if (mutatedEntities == null) { mutatedEntities = new HashMap<>(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java index 246a0bf..2d386f1 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java @@ -161,6 +161,41 @@ public class AtlasArrayType extends AtlasType { } @Override + public boolean isValidValueForUpdate(Object obj) { + if (obj != null) { + if (obj instanceof List || obj instanceof Set) { + Collection objList = (Collection) obj; + + if (!isValidElementCount(objList.size())) { + return false; + } + + for (Object element : objList) { + if (!elementType.isValidValueForUpdate(element)) { + return false; + } + } + } else if (obj.getClass().isArray()) { + int arrayLen = Array.getLength(obj); + + if (!isValidElementCount(arrayLen)) { + return false; + } + + for (int i = 0; i < arrayLen; i++) { + if (!elementType.isValidValueForUpdate(Array.get(obj, i))) { + return false; + } + } + } else { + return false; // invalid type + } + } + + return true; + } + + @Override public Collection getNormalizedValue(Object obj) { if (obj == null) { return null; @@ -222,6 +257,67 @@ public class AtlasArrayType extends AtlasType { } @Override + public Collection getNormalizedValueForUpdate(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof List || obj instanceof Set) { + List ret = new ArrayList<>(); + + Collection objList = (Collection) obj; + + if (!isValidElementCount(objList.size())) { + return null; + } + + for (Object element : objList) { + if (element != null) { + Object normalizedValue = elementType.getNormalizedValueForUpdate(element); + + if (normalizedValue != null) { + ret.add(normalizedValue); + } else { + return null; // invalid element value + } + } else { + ret.add(element); + } + } + + return ret; + } else if (obj.getClass().isArray()) { + List ret = new ArrayList<>(); + + int arrayLen = Array.getLength(obj); + + if (!isValidElementCount(arrayLen)) { + return null; + } + + for (int i = 0; i < arrayLen; i++) { + Object element = Array.get(obj, i); + + if (element != null) { + Object normalizedValue = elementType.getNormalizedValueForUpdate(element); + + if (normalizedValue != null) { + ret.add(normalizedValue); + } else { + return null; // invalid element value + } + } else { + ret.add(element); + } + } + + return ret; + } + + return null; + } + + @Override public boolean validateValue(Object obj, String objName, List messages) { boolean ret = true; @@ -265,6 +361,49 @@ public class AtlasArrayType extends AtlasType { } @Override + public boolean validateValueForUpdate(Object obj, String objName, List messages) { + boolean ret = true; + + if (obj != null) { + if (obj instanceof List || obj instanceof Set) { + Collection objList = (Collection) obj; + + if (!isValidElementCount(objList.size())) { + ret = false; + + messages.add(objName + ": incorrect number of values. found=" + objList.size() + + "; expected: minCount=" + minCount + ", maxCount=" + maxCount); + } + + int idx = 0; + for (Object element : objList) { + ret = elementType.validateValueForUpdate(element, objName + "[" + idx + "]", messages) && ret; + idx++; + } + } else if (obj.getClass().isArray()) { + int arrayLen = Array.getLength(obj); + + if (!isValidElementCount(arrayLen)) { + ret = false; + + messages.add(objName + ": incorrect number of values. found=" + arrayLen + + "; expected: minCount=" + minCount + ", maxCount=" + maxCount); + } + + for (int i = 0; i < arrayLen; i++) { + ret = elementType.validateValueForUpdate(Array.get(obj, i), objName + "[" + i + "]", messages) && ret; + } + } else { + ret = false; + + messages.add(objName + "=" + obj + ": invalid value for type " + getTypeName()); + } + } + + return ret; + } + + @Override public AtlasType getTypeForAttribute() { AtlasType elementAttributeType = elementType.getTypeForAttribute(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index 318f2ed..82b0310 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -159,6 +159,21 @@ public class AtlasClassificationType extends AtlasStructType { } @Override + public boolean isValidValueForUpdate(Object obj) { + if (obj != null) { + for (AtlasClassificationType superType : superTypes) { + if (!superType.isValidValueForUpdate(obj)) { + return false; + } + } + + return super.isValidValueForUpdate(obj); + } + + return true; + } + + @Override public Object getNormalizedValue(Object obj) { Object ret = null; @@ -178,6 +193,25 @@ public class AtlasClassificationType extends AtlasStructType { } @Override + public Object getNormalizedValueForUpdate(Object obj) { + Object ret = null; + + if (obj != null) { + if (isValidValueForUpdate(obj)) { + if (obj instanceof AtlasClassification) { + normalizeAttributeValuesForUpdate((AtlasClassification) obj); + ret = obj; + } else if (obj instanceof Map) { + normalizeAttributeValuesForUpdate((Map) obj); + ret = obj; + } + } + } + + return ret; + } + + @Override public boolean validateValue(Object obj, String objName, List messages) { boolean ret = true; @@ -192,6 +226,21 @@ public class AtlasClassificationType extends AtlasStructType { return ret; } + @Override + public boolean validateValueForUpdate(Object obj, String objName, List messages) { + boolean ret = true; + + if (obj != null) { + for (AtlasClassificationType superType : superTypes) { + ret = superType.validateValueForUpdate(obj, objName, messages) && ret; + } + + ret = super.validateValueForUpdate(obj, objName, messages) && ret; + } + + return ret; + } + public void normalizeAttributeValues(AtlasClassification classification) { if (classification != null) { for (AtlasClassificationType superType : superTypes) { @@ -202,6 +251,16 @@ public class AtlasClassificationType extends AtlasStructType { } } + public void normalizeAttributeValuesForUpdate(AtlasClassification classification) { + if (classification != null) { + for (AtlasClassificationType superType : superTypes) { + superType.normalizeAttributeValuesForUpdate(classification); + } + + super.normalizeAttributeValuesForUpdate(classification); + } + } + @Override public void normalizeAttributeValues(Map obj) { if (obj != null) { @@ -213,6 +272,16 @@ public class AtlasClassificationType extends AtlasStructType { } } + public void normalizeAttributeValuesForUpdate(Map obj) { + if (obj != null) { + for (AtlasClassificationType superType : superTypes) { + superType.normalizeAttributeValuesForUpdate(obj); + } + + super.normalizeAttributeValuesForUpdate(obj); + } + } + public void populateDefaultValues(AtlasClassification classification) { if (classification != null) { for (AtlasClassificationType superType : superTypes) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index d997725..626f18e 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -142,17 +142,26 @@ public class AtlasEntityType extends AtlasStructType { @Override public boolean isValidValue(Object obj) { if (obj != null) { - if (obj instanceof AtlasObjectId) { - AtlasObjectId objId = (AtlasObjectId ) obj; - return isAssignableFrom(objId); - } else { - for (AtlasEntityType superType : superTypes) { - if (!superType.isValidValue(obj)) { - return false; - } + for (AtlasEntityType superType : superTypes) { + if (!superType.isValidValue(obj)) { + return false; + } + } + return super.isValidValue(obj); + } + + return true; + } + + @Override + public boolean isValidValueForUpdate(Object obj) { + if (obj != null) { + for (AtlasEntityType superType : superTypes) { + if (!superType.isValidValueForUpdate(obj)) { + return false; } - return super.isValidValue(obj); } + return super.isValidValueForUpdate(obj); } return true; @@ -170,7 +179,24 @@ public class AtlasEntityType extends AtlasStructType { } else if (obj instanceof Map) { normalizeAttributeValues((Map) obj); ret = obj; - } else if (obj instanceof AtlasObjectId) { + } + } + } + + return ret; + } + + @Override + public Object getNormalizedValueForUpdate(Object obj) { + Object ret = null; + + if (obj != null) { + if (isValidValueForUpdate(obj)) { + if (obj instanceof AtlasEntity) { + normalizeAttributeValuesForUpdate((AtlasEntity) obj); + ret = obj; + } else if (obj instanceof Map) { + normalizeAttributeValuesForUpdate((Map) obj); ret = obj; } } @@ -189,30 +215,38 @@ public class AtlasEntityType extends AtlasStructType { boolean ret = true; if (obj != null) { - if (obj instanceof AtlasObjectId) { - AtlasObjectId objId = (AtlasObjectId) obj; - return isAssignableFrom(objId); - } else if (obj instanceof AtlasEntity) { - // entity validation will be done below, outside of these if/else blocks - } else if (obj instanceof Map) { - AtlasObjectId objId = new AtlasObjectId((Map) obj); - - if (isAssignableFrom(objId)) { - return true; + if (obj instanceof AtlasEntity || obj instanceof Map) { + for (AtlasEntityType superType : superTypes) { + ret = superType.validateValue(obj, objName, messages) && ret; } - // entity validation will be done below, outside of these if/else blocks + ret = super.validateValue(obj, objName, messages) && ret; + } else { ret = false; - messages.add(objName + ": invalid value type '" + obj.getClass().getName()); } + } - for (AtlasEntityType superType : superTypes) { - ret = superType.validateValue(obj, objName, messages) && ret; - } + return ret; + } - ret = super.validateValue(obj, objName, messages) && ret; + @Override + public boolean validateValueForUpdate(Object obj, String objName, List messages) { + boolean ret = true; + + if (obj != null) { + if (obj instanceof AtlasEntity || obj instanceof Map) { + for (AtlasEntityType superType : superTypes) { + ret = superType.validateValueForUpdate(obj, objName, messages) && ret; + } + + ret = super.validateValueForUpdate(obj, objName, messages) && ret; + + } else { + ret = false; + messages.add(objName + ": invalid value type '" + obj.getClass().getName()); + } } return ret; @@ -239,6 +273,16 @@ public class AtlasEntityType extends AtlasStructType { } } + public void normalizeAttributeValuesForUpdate(AtlasEntity ent) { + if (ent != null) { + for (AtlasEntityType superType : superTypes) { + superType.normalizeAttributeValuesForUpdate(ent); + } + + super.normalizeAttributeValuesForUpdate(ent); + } + } + @Override public void normalizeAttributeValues(Map obj) { if (obj != null) { @@ -250,6 +294,16 @@ public class AtlasEntityType extends AtlasStructType { } } + public void normalizeAttributeValuesForUpdate(Map obj) { + if (obj != null) { + for (AtlasEntityType superType : superTypes) { + superType.normalizeAttributeValuesForUpdate(obj); + } + + super.normalizeAttributeValuesForUpdate(obj); + } + } + public void populateDefaultValues(AtlasEntity ent) { if (ent != null) { for (AtlasEntityType superType : superTypes) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java index f0d94a9..385a9ae 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java @@ -124,6 +124,25 @@ public class AtlasMapType extends AtlasType { } @Override + public boolean isValidValueForUpdate(Object obj) { + if (obj != null) { + if (obj instanceof Map) { + Map map = (Map) obj; + + for (Map.Entry e : map.entrySet()) { + if (!keyType.isValidValueForUpdate(e.getKey()) || !valueType.isValidValueForUpdate(e.getValue())) { + return false; // invalid key/value + } + } + } else { + return false; // invalid type + } + } + + return true; + } + + @Override public Map getNormalizedValue(Object obj) { if (obj == null) { return null; @@ -163,6 +182,45 @@ public class AtlasMapType extends AtlasType { } @Override + public Map getNormalizedValueForUpdate(Object obj) { + if (obj == null) { + return null; + } + + if (obj instanceof Map) { + Map ret = new HashMap<>(); + + Map map = (Map) obj; + + for (Map.Entry e : map.entrySet()) { + Object normalizedKey = keyType.getNormalizedValueForUpdate(e.getKey()); + + if (normalizedKey != null) { + Object value = e.getValue(); + + if (value != null) { + Object normalizedValue = valueType.getNormalizedValueForUpdate(e.getValue()); + + if (normalizedValue != null) { + ret.put(normalizedKey, normalizedValue); + } else { + return null; // invalid value + } + } else { + ret.put(normalizedKey, value); + } + } else { + return null; // invalid key + } + } + + return ret; + } + + return null; + } + + @Override public boolean validateValue(Object obj, String objName, List messages) { boolean ret = true; @@ -194,6 +252,37 @@ public class AtlasMapType extends AtlasType { } @Override + public boolean validateValueForUpdate(Object obj, String objName, List messages) { + boolean ret = true; + + if (obj != null) { + if (obj instanceof Map) { + Map map = (Map) obj; + + for (Map.Entry e : map.entrySet()) { + Object key = e.getKey(); + + if (!keyType.isValidValueForUpdate(key)) { + ret = false; + + messages.add(objName + "." + key + ": invalid key for type " + getTypeName()); + } else { + Object value = e.getValue(); + + ret = valueType.validateValueForUpdate(value, objName + "." + key, messages) && ret; + } + } + } else { + ret = false; + + messages.add(objName + "=" + obj + ": invalid value for type " + getTypeName()); + } + } + + return ret; + } + + @Override public AtlasType getTypeForAttribute() { AtlasType keyAttributeType = keyType.getTypeForAttribute(); AtlasType valueAttributeType = valueType.getTypeForAttribute(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 3bc5f83..57ad106 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -199,7 +199,7 @@ public class AtlasStructType extends AtlasType { } } } else if (obj instanceof Map) { - Map map = (Map) obj; + Map map = AtlasTypeUtil.toStructAttributes((Map) obj); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { if (!isAssignableValue(map.get(attributeDef.getName()), attributeDef)) { @@ -215,6 +215,42 @@ public class AtlasStructType extends AtlasType { } @Override + public boolean isValidValueForUpdate(Object obj) { + if (obj != null) { + Map attributes; + + if (obj instanceof AtlasStruct) { + AtlasStruct structObj = (AtlasStruct) obj; + attributes = structObj.getAttributes(); + + } else if (obj instanceof Map) { + attributes = AtlasTypeUtil.toStructAttributes((Map) obj); + + } else { + return false; + } + + if (MapUtils.isNotEmpty(attributes)) { + for (Map.Entry e : attributes.entrySet()) { + String attrName = e.getKey(); + Object attrValue = e.getValue(); + AtlasAttributeDef attrDef = structDef.getAttribute(attrName); + + if (attrValue == null || attrDef == null) { + continue; + } + + if (!isAssignableValueForUpdate(attrValue, attrDef)) { + return false; + } + } + } + } + + return true; + } + + @Override public Object getNormalizedValue(Object obj) { Object ret = null; @@ -234,6 +270,25 @@ public class AtlasStructType extends AtlasType { } @Override + public Object getNormalizedValueForUpdate(Object obj) { + Object ret = null; + + if (obj != null) { + if (isValidValueForUpdate(obj)) { + if (obj instanceof AtlasStruct) { + normalizeAttributeValuesForUpdate((AtlasStruct) obj); + ret = obj; + } else if (obj instanceof Map) { + normalizeAttributeValuesForUpdate((Map) obj); + ret = obj; + } + } + } + + return ret; + } + + @Override public boolean validateValue(Object obj, String objName, List messages) { boolean ret = true; @@ -242,20 +297,18 @@ public class AtlasStructType extends AtlasType { AtlasStruct structObj = (AtlasStruct) obj; for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { - String attrName = attributeDef.getName(); - + String attrName = attributeDef.getName(); AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); if (attribute != null) { - AtlasType dataType = attribute.getAttributeType(); - Object value = structObj.getAttribute(attrName); - String fieldName = objName + "." + attrName; + AtlasType dataType = attribute.getAttributeType(); + Object value = structObj.getAttribute(attrName); + String fieldName = objName + "." + attrName; if (value != null) { ret = dataType.validateValue(value, fieldName, messages) && ret; } else if (!attributeDef.getIsOptional()) { ret = false; - messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName()); } } @@ -264,19 +317,18 @@ public class AtlasStructType extends AtlasType { Map attributes = AtlasTypeUtil.toStructAttributes((Map)obj); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { - String attrName = attributeDef.getName(); - AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); + String attrName = attributeDef.getName(); + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); if (attribute != null) { - AtlasType dataType = attribute.getAttributeType(); - Object value = attributes.get(attrName); - String fieldName = objName + "." + attrName; + AtlasType dataType = attribute.getAttributeType(); + Object value = attributes.get(attrName); + String fieldName = objName + "." + attrName; if (value != null) { ret = dataType.validateValue(value, fieldName, messages) && ret; } else if (!attributeDef.getIsOptional()) { ret = false; - messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName()); } } @@ -290,6 +342,47 @@ public class AtlasStructType extends AtlasType { return ret; } + @Override + public boolean validateValueForUpdate(Object obj, String objName, List messages) { + boolean ret = true; + Map attributes = null; + + if (obj != null) { + if (obj instanceof AtlasStruct) { + AtlasStruct structObj = (AtlasStruct) obj; + attributes = structObj.getAttributes(); + + } else if (obj instanceof Map) { + attributes = AtlasTypeUtil.toStructAttributes((Map) obj); + + } else { + ret = false; + messages.add(objName + "=" + obj + ": invalid value for type " + getTypeName()); + } + + if (MapUtils.isNotEmpty(attributes)) { + for (Map.Entry e : attributes.entrySet()) { + String attrName = e.getKey(); + Object attrValue = e.getValue(); + AtlasAttribute attribute = allAttributes.get(attrName); + + if (attrValue == null) { + continue; + } + + if (attribute != null) { + AtlasType dataType = attribute.getAttributeType(); + String fieldName = objName + "." + attrName; + + ret = dataType.validateValueForUpdate(attrValue, fieldName, messages) && ret; + } + } + } + } + + return ret; + } + public void normalizeAttributeValues(AtlasStruct obj) { if (obj != null) { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { @@ -306,6 +399,19 @@ public class AtlasStructType extends AtlasType { } } + public void normalizeAttributeValuesForUpdate(AtlasStruct obj) { + if (obj != null) { + for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { + String attributeName = attributeDef.getName(); + + if (obj.hasAttribute(attributeName)) { + Object attributeValue = getNormalizedValueForUpdate(obj.getAttribute(attributeName), attributeDef); + obj.setAttribute(attributeName, attributeValue); + } + } + } + } + public void normalizeAttributeValues(Map obj) { if (obj != null) { for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { @@ -322,6 +428,20 @@ public class AtlasStructType extends AtlasType { } } + public void normalizeAttributeValuesForUpdate(Map obj) { + if (obj != null) { + for (AtlasAttributeDef attrDef : structDef.getAttributeDefs()) { + String attrName = attrDef.getName(); + Object attrValue = obj.get(attrName); + + if (obj.containsKey(attrName)) { + attrValue = getNormalizedValueForUpdate(attrValue, attrDef); + obj.put(attrName, attrValue); + } + } + } + } + public void populateDefaultValues(AtlasStruct obj) { if (obj != null) { Map attributes = obj.getAttributes(); @@ -376,6 +496,24 @@ public class AtlasStructType extends AtlasType { return ret; } + private boolean isAssignableValueForUpdate(Object value, AtlasAttributeDef attributeDef) { + boolean ret = true; + + if (value != null) { + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); + + if (attribute != null) { + AtlasType attrType = attribute.getAttributeType(); + + if (!attrType.isValidValueForUpdate(value)) { + ret = false; // invalid value + } + } + } + + return ret; + } + private Object getNormalizedValue(Object value, AtlasAttributeDef attributeDef) { AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); @@ -394,6 +532,20 @@ public class AtlasStructType extends AtlasType { return null; } + private Object getNormalizedValueForUpdate(Object value, AtlasAttributeDef attributeDef) { + AtlasAttribute attribute = allAttributes.get(attributeDef.getName()); + + if (attribute != null) { + AtlasType attrType = attribute.getAttributeType(); + + if (value != null) { + return attrType.getNormalizedValueForUpdate(value); + } + } + + return null; + } + public String getQualifiedAttributeName(String attrName) throws AtlasBaseException { if ( allAttributes.containsKey(attrName)) { return allAttributes.get(attrName).getQualifiedName(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/main/java/org/apache/atlas/type/AtlasType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java index 59d93cf..6d0c357 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java @@ -77,6 +77,14 @@ public abstract class AtlasType { return ret; } + public boolean isValidValueForUpdate(Object obj) { return isValidValue(obj); } + + public Object getNormalizedValueForUpdate(Object obj) { return getNormalizedValue(obj); } + + public boolean validateValueForUpdate(Object obj, String objName, List messages) { + return validateValue(obj, objName, messages); + } + /* for attribute of entity-type, the value would be of AtlasObjectId * when an attribute instance is created i.e. AtlasAttribute, this method * will be called to get AtlasEntityType replaced with AtlasObjectType http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 00566a1..ea56dd6 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -582,6 +582,7 @@ public final class TestUtilsV2 { ImmutableSet.of(), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), AtlasTypeUtil.createRequiredAttrDef("type", "string"), + AtlasTypeUtil.createOptionalAttrDef("description", "string"), new AtlasAttributeDef("table", TABLE_TYPE, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index f3e9563..260c5a8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -66,10 +66,9 @@ public interface AtlasEntityStore { * @return EntityMutationResponse Entity mutations operations with the corresponding set of entities on which these operations were performed * @throws AtlasBaseException */ - EntityMutationResponse createOrUpdate(EntityStream entityStream) throws AtlasBaseException; + EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean isPartialUpdate) throws AtlasBaseException; /** - * @deprecated * Update a single entity * @param entityType type of the entity * @param uniqAttributes Attributes that uniquely identify the entity http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java index a12ba3b..23a25b9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java @@ -37,5 +37,9 @@ public interface EntityGraphDiscovery { */ EntityGraphDiscoveryContext discoverEntities() throws AtlasBaseException; + void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException; + + void validateAndNormalizeForUpdate(AtlasEntity entity) throws AtlasBaseException; + void cleanUp() throws AtlasBaseException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java index 256c5f6..436de49 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java @@ -59,9 +59,8 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { @Override public EntityGraphDiscoveryContext discoverEntities() throws AtlasBaseException { - // walk through entities in stream and validate them; record entity references - discoverAndValidate(); + discover(); // resolve entity references discovered in previous step resolveReferences(); @@ -70,12 +69,58 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { } @Override + public void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException { + List messages = new ArrayList<>(); + + if (!AtlasEntity.isAssigned(entity.getGuid()) && !AtlasEntity.isUnAssigned(entity.getGuid())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid()); + } + + AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + if (type == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); + } + + type.validateValue(entity, entity.getTypeName(), messages); + + if (!messages.isEmpty()) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages); + } + + type.getNormalizedValue(entity); + } + + @Override + public void validateAndNormalizeForUpdate(AtlasEntity entity) throws AtlasBaseException { + List messages = new ArrayList<>(); + + if (!AtlasEntity.isAssigned(entity.getGuid()) && !AtlasEntity.isUnAssigned(entity.getGuid())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid()); + } + + AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + if (type == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); + } + + type.validateValueForUpdate(entity, entity.getTypeName(), messages); + + if (!messages.isEmpty()) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages); + } + + type.getNormalizedValueForUpdate(entity); + } + + @Override public void cleanUp() throws AtlasBaseException { discoveryContext.cleanUp(); } - protected void discoverAndValidate() throws AtlasBaseException { + protected void discover() throws AtlasBaseException { EntityStream entityStream = discoveryContext.getEntityStream(); Set walkedEntities = new HashSet<>(); @@ -261,7 +306,6 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { return; } - validateAndNormalize(entity); AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName()); recordObjectReference(entity.getGuid()); @@ -274,28 +318,6 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { return typeCategory == TypeCategory.PRIMITIVE || typeCategory == TypeCategory.ENUM; } - private void validateAndNormalize(AtlasEntity entity) throws AtlasBaseException { - List messages = new ArrayList<>(); - - if (!AtlasEntity.isAssigned(entity.getGuid()) && !AtlasEntity.isUnAssigned(entity.getGuid())) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, "invalid guid " + entity.getGuid()); - } - - AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName()); - - if (type == null) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); - } - - type.validateValue(entity, entity.getTypeName(), messages); - - if (!messages.isEmpty()) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, messages); - } - - type.getNormalizedValue(entity); - } - private void recordObjectReference(String guid) { discoveryContext.addReferencedGuid(guid); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java index 34c10f4..5cb276c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java @@ -136,7 +136,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { @Override @GraphTransaction - public EntityMutationResponse createOrUpdate(EntityStream entityStream) throws AtlasBaseException { + public EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean isPartialUpdate) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> createOrUpdate()"); } @@ -145,12 +145,12 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "no entities to create/update."); } + // Create/Update entities EntityGraphMapper entityGraphMapper = new EntityGraphMapper(deleteHandler, typeRegistry); - // Create/Update entities - EntityMutationContext context = preCreateOrUpdate(entityStream, entityGraphMapper); + EntityMutationContext context = preCreateOrUpdate(entityStream, entityGraphMapper, isPartialUpdate); - EntityMutationResponse ret = entityGraphMapper.mapAttributes(context); + EntityMutationResponse ret = entityGraphMapper.mapAttributes(context, isPartialUpdate); ret.setGuidAssignments(context.getGuidAssignments()); @@ -164,8 +164,21 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { @Override @GraphTransaction public EntityMutationResponse updateByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes, - AtlasEntity entity) throws AtlasBaseException { - throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "updateByUniqueAttributes() not implemented yet"); + AtlasEntity updatedEntity) throws AtlasBaseException { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> updateByUniqueAttributes({}, {})", entityType.getTypeName(), uniqAttributes); + } + + if (updatedEntity == null) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "no entity to update."); + } + + AtlasVertex entityVertex = AtlasGraphUtilsV1.getVertexByUniqueAttributes(entityType, uniqAttributes); + + updatedEntity.setGuid(AtlasGraphUtilsV1.getIdFromVertex(entityVertex)); + + return createOrUpdate(new AtlasEntityStream(updatedEntity), true); } @GraphTransaction @@ -256,7 +269,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { } - private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, EntityGraphMapper entityGraphMapper) throws AtlasBaseException { + private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, EntityGraphMapper entityGraphMapper, boolean isPartialUpdate) throws AtlasBaseException { EntityGraphDiscovery graphDiscoverer = new AtlasEntityGraphDiscoveryV1(typeRegistry, entityStream); EntityGraphDiscoveryContext discoveryContext = graphDiscoverer.discoverEntities(); EntityMutationContext context = new EntityMutationContext(discoveryContext); @@ -265,9 +278,16 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { AtlasVertex vertex = discoveryContext.getResolvedEntityVertex(guid); AtlasEntity entity = entityStream.getByGuid(guid); - if (vertex != null) { - // entity would be null if guid is not in the stream but referenced by an entity in the stream - if (entity != null) { + if (entity != null) { + + if (vertex != null) { + // entity would be null if guid is not in the stream but referenced by an entity in the stream + if (!isPartialUpdate) { + graphDiscoverer.validateAndNormalize(entity); + } else { + graphDiscoverer.validateAndNormalizeForUpdate(entity); + } + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); String guidVertex = AtlasGraphUtilsV1.getIdFromVertex(vertex); @@ -277,21 +297,22 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { } context.addUpdated(guid, entity, entityType, vertex); - } - } else { - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + } else { + graphDiscoverer.validateAndNormalize(entity); - //Create vertices which do not exist in the repository - vertex = entityGraphMapper.createVertex(entity); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); - discoveryContext.addResolvedGuid(guid, vertex); + //Create vertices which do not exist in the repository + vertex = entityGraphMapper.createVertex(entity); - String generatedGuid = AtlasGraphUtilsV1.getIdFromVertex(vertex); + discoveryContext.addResolvedGuid(guid, vertex); - entity.setGuid(generatedGuid); + String generatedGuid = AtlasGraphUtilsV1.getIdFromVertex(vertex); - context.addCreated(guid, entity, entityType, vertex); + entity.setGuid(generatedGuid); + context.addCreated(guid, entity, entityType, vertex); + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index 6a1b301..397ee7e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -63,6 +63,7 @@ import java.util.Set; import java.util.UUID; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CREATE; +import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE; @@ -103,7 +104,7 @@ public class EntityGraphMapper { return ret; } - public EntityMutationResponse mapAttributes(EntityMutationContext context) throws AtlasBaseException { + public EntityMutationResponse mapAttributes(EntityMutationContext context, boolean isPartialUpdate) throws AtlasBaseException { EntityMutationResponse resp = new EntityMutationResponse(); Collection createdEntities = context.getCreatedEntities(); @@ -129,7 +130,12 @@ public class EntityGraphMapper { mapAttributes(updatedEntity, vertex, UPDATE, context); - resp.addEntity(UPDATE, constructHeader(updatedEntity, entityType, vertex)); + if (isPartialUpdate) { + resp.addEntity(PARTIAL_UPDATE, constructHeader(updatedEntity, entityType, vertex)); + } else { + resp.addEntity(UPDATE, constructHeader(updatedEntity, entityType, vertex)); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/68c55925/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java index b35d288..9653b72 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasDeleteHandlerV1Test.java @@ -163,7 +163,7 @@ public abstract class AtlasDeleteHandlerV1Test { public void testDeleteAndCreate() throws Exception { init(); final AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity)); + EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); init(); //delete entity should mark it as deleted @@ -182,7 +182,7 @@ public abstract class AtlasDeleteHandlerV1Test { init(); //Create the same entity again, should create new entity AtlasEntity newDBEntity = TestUtilsV2.createDBEntity((String) dbEntity.getAttribute(NAME)); - EntityMutationResponse newCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(newDBEntity)); + EntityMutationResponse newCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(newDBEntity), false); assertNotEquals(newCreationResponse.getFirstEntityCreated().getGuid(), response.getFirstEntityCreated().getGuid()); //TODO - Enable after GET is ready @@ -197,7 +197,7 @@ public abstract class AtlasDeleteHandlerV1Test { final AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); init(); - EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity)); + EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); final AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity); final AtlasEntity columnEntity = TestUtilsV2.createColumnEntity(tableEntity); @@ -207,7 +207,7 @@ public abstract class AtlasDeleteHandlerV1Test { input.addReferredEntity(columnEntity); init(); - EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(input)); + EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(input), false); final AtlasEntityHeader columnCreated = tblCreationResponse.getFirstCreatedEntityByTypeName(COLUMN_TYPE); final AtlasEntityHeader tableCreated = tblCreationResponse.getFirstCreatedEntityByTypeName(TABLE_TYPE); @@ -227,7 +227,7 @@ public abstract class AtlasDeleteHandlerV1Test { //Deleting table should update process AtlasEntity process = TestUtilsV2.createProcessEntity(null, Arrays.asList(tableCreated.getAtlasObjectId())); init(); - final EntityMutationResponse processCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(process)); + final EntityMutationResponse processCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(process), false); init(); entityStore.deleteById(tableCreated.getGuid()); @@ -242,7 +242,7 @@ public abstract class AtlasDeleteHandlerV1Test { // Create a table entity, with 3 composite column entities init(); final AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity)); + EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); final AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity); AtlasEntity.AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(tableEntity); @@ -258,7 +258,7 @@ public abstract class AtlasDeleteHandlerV1Test { init(); - final EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo)); + final EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); final AtlasEntityHeader column1Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity1.getAttribute(NAME)); final AtlasEntityHeader column2Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity2.getAttribute(NAME)); @@ -296,7 +296,7 @@ public abstract class AtlasDeleteHandlerV1Test { entitiesInfo1.addReferredEntity(columnEntity3New); init(); - deletionResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo1)); + deletionResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo1), false); //TODO - enable after fixing unique atribute resolver assertEquals(deletionResponse.getDeletedEntities().size(), 1); @@ -347,7 +347,7 @@ public abstract class AtlasDeleteHandlerV1Test { init(); RequestContextV1.clear(); - final EntityMutationResponse hrDeptCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(hrDept)); + final EntityMutationResponse hrDeptCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(hrDept), false); final AtlasEntityHeader deptCreated = hrDeptCreationResponse.getFirstUpdatedEntityByTypeName(DEPARTMENT_TYPE); final AtlasEntityHeader maxEmployeeCreated = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.EMPLOYEE_TYPE, NAME, "Max"); final AtlasEntityHeader johnEmployeeCreated = hrDeptCreationResponse.getUpdatedEntityByTypeNameAndAttribute(TestUtilsV2.EMPLOYEE_TYPE, NAME, "John"); @@ -372,7 +372,7 @@ public abstract class AtlasDeleteHandlerV1Test { maxEmployee.setAttribute("manager", janeEmployeeCreated.getAtlasObjectId()); init(); - EntityMutationResponse entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee)); + EntityMutationResponse entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee), false); assertEquals(entityResult.getUpdatedEntities().size(), 1); assertTrue(extractGuids(entityResult.getUpdatedEntities()).contains(maxGuid)); @@ -391,7 +391,7 @@ public abstract class AtlasDeleteHandlerV1Test { // Update max's mentor reference to jane. maxEmployee.setAttribute("mentor", janeEmployeeCreated.getAtlasObjectId()); init(); - entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee)); + entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee), false); assertEquals(entityResult.getUpdatedEntities().size(), 1); assertTrue(extractGuids(entityResult.getUpdatedEntities()).contains(maxGuid)); @@ -411,7 +411,7 @@ public abstract class AtlasDeleteHandlerV1Test { init(); maxEmployee.setAttribute("manager", juliusEmployeeCreated.getAtlasObjectId()); - entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee)); + entityResult = entityStore.createOrUpdate(new AtlasEntityStream(maxEmployee), false); //TODO ATLAS-499 should have updated julius' subordinates assertEquals(entityResult.getUpdatedEntities().size(), 2); assertTrue(extractGuids(entityResult.getUpdatedEntities()).contains(maxGuid)); @@ -464,7 +464,7 @@ public abstract class AtlasDeleteHandlerV1Test { public void testDisconnectBidirectionalReferences() throws Exception { AtlasEntity.AtlasEntitiesWithExtInfo hrDept = TestUtilsV2.createDeptEg2(); init(); - final EntityMutationResponse hrDeptCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(hrDept)); + final EntityMutationResponse hrDeptCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(hrDept), false); final AtlasEntityHeader deptCreated = hrDeptCreationResponse.getFirstCreatedEntityByTypeName(DEPARTMENT_TYPE); final AtlasEntityHeader maxEmployee = hrDeptCreationResponse.getCreatedEntityByTypeNameAndAttribute(TestUtilsV2.EMPLOYEE_TYPE, NAME, "Max"); @@ -647,7 +647,7 @@ public abstract class AtlasDeleteHandlerV1Test { AtlasEntityStream entityStream = new AtlasEntityStream(structCreationObj); - EntityMutationResponse response = entityStore.createOrUpdate(entityStream); + EntityMutationResponse response = entityStore.createOrUpdate(entityStream, false); Assert.assertEquals(response.getCreatedEntities().size(), 3); final List structTarget = metadataService.getEntityList("StructTarget"); @@ -726,7 +726,7 @@ public abstract class AtlasDeleteHandlerV1Test { // Create a table entity, with 3 composite column entities init(); final AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); - EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity)); + EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); final AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity); AtlasEntity.AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(tableEntity); @@ -742,7 +742,7 @@ public abstract class AtlasDeleteHandlerV1Test { init(); - final EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo)); + final EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); final AtlasEntityHeader column1Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity1.getAttribute(NAME)); final AtlasEntityHeader column2Created = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(COLUMN_TYPE, NAME, (String) columnEntity2.getAttribute(NAME));