Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8CA4A200C17 for ; Fri, 27 Jan 2017 01:04:46 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 8AFF5160B50; Fri, 27 Jan 2017 00:04:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 9695F160B4C for ; Fri, 27 Jan 2017 01:04:44 +0100 (CET) Received: (qmail 80918 invoked by uid 500); 27 Jan 2017 00:04:43 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 80909 invoked by uid 99); 27 Jan 2017 00:04:43 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Jan 2017 00:04:43 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 3FFE918C8E4 for ; Fri, 27 Jan 2017 00:04:43 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.219 X-Spam-Level: X-Spam-Status: No, score=-6.219 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id KkrjjSIOgCxP for ; Fri, 27 Jan 2017 00:04:31 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id 03A2B5FC4A for ; Fri, 27 Jan 2017 00:04:29 +0000 (UTC) Received: (qmail 79921 invoked by uid 99); 27 Jan 2017 00:04:29 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Jan 2017 00:04:29 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 707AADFD73; Fri, 27 Jan 2017 00:04:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@atlas.incubator.apache.org Date: Fri, 27 Jan 2017 00:04:30 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] incubator-atlas git commit: ATLAS-1498: added unit-tests to validate handling of array/map/struct attributes in entity create/update archived-at: Fri, 27 Jan 2017 00:04:46 -0000 ATLAS-1498: added unit-tests to validate handling of array/map/struct attributes in entity create/update Signed-off-by: Madhan Neethiraj Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/6ccba52c Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/6ccba52c Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/6ccba52c Branch: refs/heads/master Commit: 6ccba52c293d4c7e419a63326665faee7db9f231 Parents: 19a5f65 Author: Suma Shivaprasad Authored: Thu Jan 26 16:03:12 2017 -0800 Committer: Madhan Neethiraj Committed: Thu Jan 26 16:04:19 2017 -0800 ---------------------------------------------------------------------- .../java/org/apache/atlas/AtlasErrorCode.java | 2 +- .../atlas/model/instance/AtlasStruct.java | 3 +- .../model/instance/EntityMutationResponse.java | 53 +- .../apache/atlas/type/AtlasBuiltInTypes.java | 6 + .../org/apache/atlas/type/AtlasMapType.java | 6 +- .../org/apache/atlas/type/AtlasStructType.java | 1 + .../java/org/apache/atlas/type/AtlasType.java | 6 +- .../test/java/org/apache/atlas/TestUtilsV2.java | 14 +- release-log.txt | 1 + .../store/graph/v1/ArrayVertexMapper.java | 2 +- .../graph/v1/AtlasEntityGraphDiscoveryV1.java | 51 +- .../store/graph/v1/AtlasEntityStoreV1.java | 94 ++-- .../store/graph/v1/AtlasGraphUtilsV1.java | 6 +- .../store/graph/v1/AtlasStructDefStoreV1.java | 3 +- .../store/graph/v1/DeleteHandlerV1.java | 8 +- .../store/graph/v1/EntityGraphMapper.java | 48 +- .../store/graph/v1/EntityMutationContext.java | 9 +- .../store/graph/v1/GraphMutationContext.java | 27 +- .../store/graph/v1/IDBasedEntityResolver.java | 2 +- .../store/graph/v1/MapVertexMapper.java | 9 +- .../store/graph/v1/StructVertexMapper.java | 107 ++-- .../graph/v1/UniqAttrBasedEntityResolver.java | 8 +- .../store/graph/v1/AtlasEntityStoreV1Test.java | 541 +++++++++++++++++-- .../adapters/AtlasEntityFormatConverter.java | 4 +- .../web/adapters/AtlasInstanceRestAdapters.java | 13 +- .../atlas/web/adapters/TestEntitiesREST.java | 13 +- 26 files changed, 812 insertions(+), 225 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index e4f3dfd..e7dbb1c 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -60,7 +60,7 @@ public enum AtlasErrorCode { TYPE_NAME_NOT_FOUND(404, "ATLAS4041E", "Given typename {0} was invalid"), TYPE_GUID_NOT_FOUND(404, "ATLAS4042E", "Given type guid {0} was invalid"), EMPTY_RESULTS(404, "ATLAS4044E", "No result found for {0}"), - INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid"), + INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid/not found"), INSTANCE_LINEAGE_INVALID_PARAMS(404, "ATLAS4046E", "Invalid lineage query parameters passed {0}: {1}"), INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"), DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java index 794310a..c23f4d5 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java @@ -146,8 +146,9 @@ public class AtlasStruct implements Serializable { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AtlasStruct that = (AtlasStruct) o; + return Objects.equals(typeName, that.typeName) && - Objects.equals(attributes, that.attributes); + Objects.equals(attributes, that.attributes); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java index 8aba1fb..c9b6e97 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java @@ -18,9 +18,11 @@ package org.apache.atlas.model.instance; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnore; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -43,7 +45,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL @XmlAccessorType(XmlAccessType.PROPERTY) public class EntityMutationResponse { - Map> entitiesMutated = new HashMap<>(); + Map> entitiesMutated; public EntityMutationResponse() { } @@ -67,15 +69,39 @@ public class EntityMutationResponse { return null; } + @JsonIgnore + public AtlasEntityHeader getFirstEntityCreated() { + final List entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.CREATE); + if ( entitiesByOperation != null && entitiesByOperation.size() > 0) { + return entitiesByOperation.get(0); + } + + return null; + } + + @JsonIgnore + public AtlasEntityHeader getFirstEntityUpdated() { + final List entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE); + if ( entitiesByOperation != null && entitiesByOperation.size() > 0) { + return entitiesByOperation.get(0); + } + + return null; + } + public void addEntity(EntityMutations.EntityOperation op, AtlasEntityHeader header) { if (entitiesMutated == null) { entitiesMutated = new HashMap<>(); } - if (entitiesMutated != null && entitiesMutated.get(op) == null) { - entitiesMutated.put(op, new ArrayList()); + List opEntities = entitiesMutated.get(op); + + if (opEntities == null) { + opEntities = new ArrayList<>(); + entitiesMutated.put(op, opEntities); } - entitiesMutated.get(op).add(header); + + opEntities.add(header); } @@ -84,24 +110,7 @@ public class EntityMutationResponse { sb = new StringBuilder(); } - if (MapUtils.isNotEmpty(entitiesMutated)) { - int i = 0; - for (Map.Entry> e : entitiesMutated.entrySet()) { - if (i > 0) { - sb.append(","); - } - sb.append(e.getKey()).append(":"); - if (CollectionUtils.isNotEmpty(e.getValue())) { - for (int j = 0; i < e.getValue().size(); j++) { - if (j > 0) { - sb.append(","); - } - e.getValue().get(i).toString(sb); - } - } - i++; - } - } + AtlasBaseTypeDef.dumpObjects(entitiesMutated, sb); return sb; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java index 5ad3a55..a25f3f6 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java @@ -477,6 +477,7 @@ public class AtlasBuiltInTypes { */ public static class AtlasStringType extends AtlasType { private static final String DEFAULT_VALUE = ""; + private static final String OPTIONAL_DEFAULT_VALUE = null; public AtlasStringType() { super(AtlasBaseTypeDef.ATLAS_TYPE_STRING, TypeCategory.PRIMITIVE); @@ -488,6 +489,11 @@ public class AtlasBuiltInTypes { } @Override + public Object createOptionalDefaultValue() { + return OPTIONAL_DEFAULT_VALUE; + } + + @Override public boolean isValidValue(Object obj) { return true; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java index 6e6c522..7624bc1 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasMapType.java @@ -95,7 +95,11 @@ public class AtlasMapType extends AtlasType { public Map createDefaultValue() { Map ret = new HashMap<>(); - ret.put(keyType.createDefaultValue(), valueType.createDefaultValue()); + Object key = keyType.createDefaultValue(); + + if ( key != null) { + ret.put(key, valueType.createDefaultValue()); + } return ret; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 4712508..bac5800 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -328,6 +328,7 @@ public class AtlasStructType extends AtlasType { if (attribute != null) { AtlasType dataType = attribute.getAttributeType(); + ret = dataType.createDefaultValue(); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/main/java/org/apache/atlas/type/AtlasType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java index 6ea34b3..de71d11 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java @@ -33,7 +33,7 @@ import java.util.List; public abstract class AtlasType { private static final Gson GSON = - new GsonBuilder().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create(); + new GsonBuilder().serializeNulls().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create(); private final String typeName; private final TypeCategory typeCategory; @@ -59,6 +59,10 @@ public abstract class AtlasType { public abstract Object createDefaultValue(); + public Object createOptionalDefaultValue() { + return createDefaultValue(); + } + public abstract boolean isValidValue(Object obj); public abstract Object getNormalizedValue(Object obj); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index f9040f3..4f2916c 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -71,7 +71,7 @@ public final class TestUtilsV2 { new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0", Arrays.asList( new AtlasEnumElementDef("L1", "Element"+_description, 1), - new AtlasEnumElementDef("L1", "Element"+_description, 2) + new AtlasEnumElementDef("L2", "Element"+_description, 2) )); AtlasStructDef addressDetails = @@ -149,7 +149,7 @@ public final class TestUtilsV2 { new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0", Arrays.asList( new AtlasEnumElementDef("L1", "Element"+ _description, 1), - new AtlasEnumElementDef("L1", "Element"+ _description, 2) + new AtlasEnumElementDef("L2", "Element"+ _description, 2) )); AtlasStructDef addressDetails = @@ -443,6 +443,7 @@ public final class TestUtilsV2 { AtlasEntityDef databaseTypeDefinition = AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME), AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"), + AtlasTypeUtil.createOptionalAttrDef("isReplicated", "boolean"), AtlasTypeUtil.createOptionalAttrDef("created", "string"), AtlasTypeUtil.createRequiredAttrDef("description", "string")); @@ -543,7 +544,7 @@ public final class TestUtilsV2 { AtlasEntityDef tableTypeDefinition = AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"), - AtlasTypeUtil.createRequiredAttrDef("description", "string"), + AtlasTypeUtil.createOptionalAttrDef("description", "string"), AtlasTypeUtil.createRequiredAttrDef("type", "string"), AtlasTypeUtil.createOptionalAttrDef("created", "date"), // enum @@ -582,6 +583,11 @@ public final class TestUtilsV2 { AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, false, Collections.emptyList()), + + // new ArrayList() {{ + //add(new AtlasStructDef.AtlasConstraintDef( + // AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF, new HashMap())); + //}}), //map of structs new AtlasAttributeDef("partitionsMap", String.format("map<%s,%s>", "string", "partition_struct_type"), @@ -646,7 +652,7 @@ public final class TestUtilsV2 { entity.setAttribute("description", "random table"); entity.setAttribute("type", "type"); entity.setAttribute("tableType", "MANAGED"); - entity.setAttribute("database", dbId); + entity.setAttribute("database", new AtlasObjectId(DATABASE_TYPE, dbId)); entity.setAttribute("created", new Date()); Map partAttributes = new HashMap() {{ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a293bf1..108d86b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj) ATLAS-1114 Performance improvements for create/update entity (jnhagelb) ATLAS-1403 Perf and stability improvements to DSL search and lineage query execution (sarath.kum4r@gmail.com via mneethiraj) ATLAS-1425 Integrate Discovery/Search API in Atlas UI (kevalbhatt via mneethiraj) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java index 528430c..e63ea6e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java @@ -82,7 +82,7 @@ public class ArrayVertexMapper implements InstanceGraphMapper { Optional existingEdge = getEdgeAt(currentElements, index, arrType.getElementType()); - GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getAttribute(), + GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(), arrType.getElementType(), newElements.get(index)) .referringVertex(ctx.getReferringVertex()) .edge(existingEdge) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java index b874c5d..4f5b4f4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java @@ -18,8 +18,8 @@ package org.apache.atlas.repository.store.graph.v1; import atlas.shaded.hbase.guava.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.inject.Provider; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; @@ -112,7 +112,11 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { protected void discover(final List entities) throws AtlasBaseException { for (AtlasEntity entity : entities) { - AtlasType type = typeRegistry.getType(entity.getTypeName()); + AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + if (type == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); + } discoveredEntities.addRootEntity(entity); walkEntityGraph(type, entity); @@ -120,27 +124,24 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { } private void visitReference(AtlasEntityType type, Object entity, boolean isManagedEntity) throws AtlasBaseException { - if ( entity != null) { - if ( entity instanceof String ) { - String guid = (String) entity; - discoveredEntities.addUnResolvedIdReference(type, guid); - } else if ( entity instanceof AtlasObjectId ) { + if (entity != null) { + if (entity instanceof AtlasObjectId) { final String guid = ((AtlasObjectId) entity).getGuid(); discoveredEntities.addUnResolvedIdReference(type, guid); - } else if ( entity instanceof AtlasEntity ) { - AtlasEntity entityObj = ( AtlasEntity ) entity; - if (!processedIds.contains(entityObj.getGuid())) { - processedIds.add(entityObj.getGuid()); + } else if (entity instanceof AtlasEntity) { + AtlasEntity entityObj = (AtlasEntity) entity; + if (isManagedEntity) { + if (!processedIds.contains(entityObj.getGuid())) { + processedIds.add(entityObj.getGuid()); - if ( isManagedEntity ) { discoveredEntities.addRootEntity(entityObj); visitStruct(type, entityObj); - } else if ( entity instanceof AtlasObjectId) { - discoveredEntities.addUnResolvedIdReference(type, ((AtlasObjectId) entity).getGuid()); - } else { - discoveredEntities.addUnResolvedEntityReference(entityObj); } + } else { + discoveredEntities.addUnResolvedEntityReference(entityObj); } + } else { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, "Invalid object type " + entity.getClass()); } } } @@ -161,12 +162,14 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { } else if (attrType.getTypeCategory() == TypeCategory.STRUCT) { visitStruct(attrType, val); } else if (attrType.getTypeCategory() == TypeCategory.ENTITY) { - if ( val instanceof AtlasObjectId || val instanceof String) { + if ( val instanceof AtlasObjectId) { visitReference((AtlasEntityType) attrType, val, false); } else if ( val instanceof AtlasEntity ) { //TODO - Change this to foreign key checks after changes in the model if ( parentType.isMappedFromRefAttribute(attrDef.getName())) { visitReference((AtlasEntityType) attrType, val, true); + } else { + visitReference((AtlasEntityType) attrType, val, false); } } } @@ -179,10 +182,9 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { } if (val != null) { - Iterator it = null; + if (Map.class.isAssignableFrom(val.getClass())) { - it = ((Map) val).entrySet().iterator(); - ImmutableMap.Builder b = ImmutableMap.builder(); + Iterator it = ((Map) val).entrySet().iterator(); while (it.hasNext()) { Map.Entry e = it.next(); visitAttribute(parentType, keyType, attrDef, e.getKey()); @@ -224,11 +226,10 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery { AtlasStructType structType = (AtlasStructType) type; - for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) { - String attrName = attributeDef.getName(); - AtlasType attrType = structType.getAttributeType(attrName); - Object attrVal = ((AtlasStruct) val).getAttribute(attrName); - visitAttribute(structType, attrType, attributeDef, attrVal); + for (AtlasStructType.AtlasAttribute attribute : structType.getAllAttributes().values()) { + AtlasType attrType = attribute.getAttributeType(); + Object attrVal = ((AtlasStruct) val).getAttribute(attribute.getAttributeDef().getName()); + visitAttribute(structType, attrType, attribute.getAttributeDef(), attrVal); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java index 18e397b..48bb77e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java @@ -70,43 +70,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { return createOrUpdate(new ArrayList() {{ add(entity); }}); } - public EntityMutationContext preCreateOrUpdate(final List atlasEntities) throws AtlasBaseException { - - EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities); - EntityMutationContext context = new EntityMutationContext(discoveredEntities); - for (AtlasEntity entity : discoveredEntities.getRootEntities()) { - - AtlasVertex vertex = null; - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity); - } - - AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName()); - - if ( discoveredEntities.isResolved(entity.getGuid()) ) { - vertex = discoveredEntities.getResolvedReference(entity.getGuid()); - context.addUpdated(entity, entityType, vertex); - - String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex); - RequestContextV1.get().recordEntityUpdate(guid); - } else { - //Create vertices which do not exist in the repository - vertex = graphMapper.createVertexTemplate(entity, entityType); - context.addCreated(entity, entityType, vertex); - discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex); - - String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex); - RequestContextV1.get().recordEntityCreate(guid); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex); - } - } - - return context; - } - @Override public EntityMutationResponse updateById(final String guid, final AtlasEntity entity) { return null; @@ -205,15 +168,56 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { return null; } + private EntityMutationContext preCreateOrUpdate(final List atlasEntities) throws AtlasBaseException { + + EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities); + EntityMutationContext context = new EntityMutationContext(discoveredEntities); + for (AtlasEntity entity : discoveredEntities.getRootEntities()) { + + AtlasVertex vertex = null; + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity); + } + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + if ( entityType == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); + } + + if ( discoveredEntities.isResolved(entity.getGuid()) ) { + vertex = discoveredEntities.getResolvedReference(entity.getGuid()); + context.addUpdated(entity, entityType, vertex); + + String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex); + RequestContextV1.get().recordEntityUpdate(guid); + } else { + //Create vertices which do not exist in the repository + vertex = graphMapper.createVertexTemplate(entity, entityType); + context.addCreated(entity, entityType, vertex); + discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex); + + String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex); + RequestContextV1.get().recordEntityCreate(guid); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex); + } + } + + return context; + } + private List validateAndNormalize(final List entities) throws AtlasBaseException { List normalizedEntities = new ArrayList<>(); List messages = new ArrayList<>(); for (AtlasEntity entity : entities) { - AtlasType type = typeRegistry.getType(entity.getTypeName()); - if (type.getTypeCategory() != TypeCategory.ENTITY) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, type.getTypeCategory().name(), TypeCategory.ENTITY.name()); + AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName()); + if (type == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); } type.validateValue(entity, entity.getTypeName(), messages); @@ -232,16 +236,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore { return normalizedEntities; } - @VisibleForTesting - EntityGraphDiscovery getGraphDiscoverer() { - return graphDiscoverer; - } - - @VisibleForTesting - void setGraphDiscoverer(EntityGraphDiscovery discoverer) { - this.graphDiscoverer = discoverer; - } - public void cleanUp() throws AtlasBaseException { this.graphDiscoverer.cleanUp(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java index 1947855..7fe5e14 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java @@ -94,8 +94,12 @@ public class AtlasGraphUtilsV1 { return PROPERTY_PREFIX + "edge." + fromNode + "." + toNode; } + public static String getEdgeLabel(String property) { + return GraphHelper.EDGE_LABEL_PREFIX + property; + } + public static String getAttributeEdgeLabel(AtlasStructType fromType, String attributeName) throws AtlasBaseException { - return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedAttributePropertyKey(fromType, attributeName); + return getEdgeLabel(getQualifiedAttributePropertyKey(fromType, attributeName)); } public static String getQualifiedAttributePropertyKey(AtlasStructType fromType, String attributeName) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index 425bde9..163ddef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -25,6 +25,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasStructDefStore; import org.apache.atlas.repository.util.FilterUtil; @@ -462,7 +463,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At if (CollectionUtils.isNotEmpty(attrNames)) { for (String attrName : attrNames) { String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(ret, attrName); - String attribJson = vertex.getProperty(propertyKey, String.class); + String attribJson = vertex.getProperty(GraphHelper.encodePropertyKey(propertyKey), String.class); attributeDefs.add(toAttributeDefFromJson(structDef, AtlasType.fromJson(attribJson, Map.class), typeDefStore)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index 3b557e6..34e02c6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -18,6 +18,7 @@ package org.apache.atlas.repository.store.graph.v1; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContextV1; import org.apache.atlas.exception.AtlasBaseException; @@ -129,7 +130,12 @@ public abstract class DeleteHandlerV1 { continue; } result.add(new GraphHelper.VertexInfo(guid, vertex, typeName)); - AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(typeName); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + + if (entityType == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); + } + for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) { if (!entityType.isMappedFromRefAttribute(attributeInfo.getAttributeDef().getName())) { continue; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index 174e490..c98d340 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -22,6 +22,7 @@ import com.google.inject.Inject; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContextV1; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -36,9 +37,11 @@ import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; import java.util.UUID; public class EntityGraphMapper implements InstanceGraphMapper { @@ -86,13 +89,12 @@ public class EntityGraphMapper implements InstanceGraphMapper { AtlasEdge result = null; String guid = getId(ctx.getValue()); - AtlasVertex entityVertex = context.getDiscoveryContext().getResolvedReference(guid); - String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName()); if ( ctx.getCurrentEdge().isPresent() ) { updateEdge(ctx.getAttributeDef(), ctx.getValue(), ctx.getCurrentEdge().get(), entityVertex); result = ctx.getCurrentEdge().get(); - } else { + } else if (ctx.getValue() != null) { + String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey()); try { result = graphHelper.getOrCreateEdge(ctx.getReferringVertex(), entityVertex, edgeLabel); } catch (RepositoryException e) { @@ -112,7 +114,7 @@ public class EntityGraphMapper implements InstanceGraphMapper { LOG.debug("Updating entity reference {} for reference attribute {}", attributeDef.getName()); // Update edge if it exists - AtlasVertex currentVertex = currentEdge.getOutVertex(); + AtlasVertex currentVertex = currentEdge.getInVertex(); String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentVertex); String newEntityId = getId(value); AtlasEdge newEdge = currentEdge; @@ -140,17 +142,17 @@ public class EntityGraphMapper implements InstanceGraphMapper { if (ctx.getCreatedEntities() != null) { for (AtlasEntity createdEntity : ctx.getCreatedEntities()) { AtlasVertex vertex = ctx.getVertex(createdEntity); - structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(createdEntity), createdEntity, vertex); - resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, vertex)); + structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, ctx.getType(createdEntity), createdEntity, vertex); + resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, ctx.getType(createdEntity), vertex)); } } if (ctx.getUpdatedEntities() != null) { for (AtlasEntity updated : ctx.getUpdatedEntities()) { AtlasVertex vertex = ctx.getVertex(updated); - structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(updated), updated, vertex); + structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.UPDATE, ctx.getType(updated), updated, vertex); - resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, vertex)); + resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, ctx.getType(updated), vertex)); } } @@ -165,13 +167,30 @@ public class EntityGraphMapper implements InstanceGraphMapper { } else if (value instanceof AtlasEntity) { return ((AtlasEntity) value).getGuid(); } + + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value); } - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value); + + return null; } - private AtlasEntityHeader constructHeader(AtlasEntity entity, AtlasVertex vertex) { + private AtlasEntityHeader constructHeader(AtlasEntity entity, final AtlasEntityType type, AtlasVertex vertex) { //TODO - enhance to return only selective attributes - return new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes()); + AtlasEntityHeader header = new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes()); + final Map allAttributes = type.getAllAttributes(); + for (String attribute : allAttributes.keySet()) { + AtlasType attributeType = allAttributes.get(attribute).getAttributeType(); + AtlasStructDef.AtlasAttributeDef attributeDef = allAttributes.get(attribute).getAttributeDef(); + if ( header.getAttribute(attribute) == null && (TypeCategory.PRIMITIVE == attributeType.getTypeCategory())) { + + if ( attributeDef.getIsOptional()) { + header.setAttribute(attribute, attributeType.createOptionalDefaultValue()); + } else { + header.setAttribute(attribute, attributeType.createDefaultValue()); + } + } + } + return header; } public EntityMutationContext getContext() { @@ -180,6 +199,11 @@ public class EntityGraphMapper implements InstanceGraphMapper { public AtlasEntityType getInstanceType(Object val) throws AtlasBaseException { String guid = getId(val); - return (AtlasEntityType) getContext().getType(guid); + + if ( guid != null) { + return (AtlasEntityType) getContext().getType(guid); + } + + return null; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java index f942a91..f6e5055 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java @@ -21,6 +21,7 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasType; import java.util.ArrayList; @@ -35,20 +36,20 @@ public class EntityMutationContext { private List entitiesUpdated = new ArrayList<>(); private EntityGraphDiscoveryContext context; - private Map entityVsType = new HashMap<>(); + private Map entityVsType = new HashMap<>(); private Map entityVsVertex = new HashMap<>(); public EntityMutationContext(final EntityGraphDiscoveryContext context) { this.context = context; } - public void addCreated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) { + public void addCreated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) { entitiesCreated.add(entity); entityVsVertex.put(entity.getGuid(), atlasVertex); entityVsType.put(entity.getGuid(), type); } - public void addUpdated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) { + public void addUpdated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) { entitiesUpdated.add(entity); entityVsVertex.put(entity.getGuid(), atlasVertex); entityVsType.put(entity.getGuid(), type); @@ -62,7 +63,7 @@ public class EntityMutationContext { return entitiesUpdated; } - public AtlasType getType(AtlasEntity entity) { + public AtlasEntityType getType(AtlasEntity entity) { return entityVsType.get(entity.getGuid()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java index d5ba7e1..89419d9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v1; import com.google.common.base.Optional; +import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -30,6 +31,7 @@ import java.util.Objects; public class GraphMutationContext { + private EntityMutations.EntityOperation op; /** * Atlas Attribute */ @@ -65,6 +67,7 @@ public class GraphMutationContext { private GraphMutationContext(final Builder builder) { + this.op = builder.op; this.attribute = builder.attribute; this.currentElementType = builder.elementType; this.existingEdge = builder.currentEdge; @@ -79,7 +82,7 @@ public class GraphMutationContext { @Override public int hashCode() { - return Objects.hash(attribute, value, referringVertex, vertexPropertyKey, existingEdge); + return Objects.hash(op, attribute, value, referringVertex, vertexPropertyKey, existingEdge); } @Override @@ -96,13 +99,16 @@ public class GraphMutationContext { && Objects.equals(value, rhs.getValue()) && Objects.equals(referringVertex, rhs.getReferringVertex()) && Objects.equals(vertexPropertyKey, rhs.getReferringVertex()) - && Objects.equals(existingEdge, rhs.getCurrentEdge()); + && Objects.equals(existingEdge, rhs.getCurrentEdge()) + && Objects.equals(op, rhs.getOp()); } } public static final class Builder { + private final EntityMutations.EntityOperation op; + private final AtlasStructType.AtlasAttribute attribute; private final AtlasType elementType; @@ -116,16 +122,15 @@ public class GraphMutationContext { private String vertexPropertyKey; - public Builder(AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) { + public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) { + this.op = op; this.attribute = attribute; this.elementType = currentElementType; this.currentValue = currentValue; } - public Builder(AtlasStructType.AtlasAttribute attribute, Object currentValue) { - this.attribute = attribute; - this.elementType = null; - this.currentValue = currentValue; + public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, Object currentValue) { + this(op, attribute, null, currentValue); } Builder referringVertex(AtlasVertex referringVertex) { @@ -192,4 +197,12 @@ public class GraphMutationContext { public AtlasStructType.AtlasAttribute getAttribute() { return attribute; } + + public EntityMutations.EntityOperation getOp() { + return op; + } + + public void setExistingEdge(final Optional existingEdge) { + this.existingEdge = existingEdge; + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java index 488f141..d02b5a1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java @@ -105,7 +105,7 @@ public class IDBasedEntityResolver implements EntityResolver { if ( vertex != null ) { return Optional.of(vertex); } else { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, "Could not find an entity with the specified guid " + typeIdPair.getGuid() + " in Atlas respository"); + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, typeIdPair.getGuid()); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java index 9d219f5..efafcc9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java @@ -97,13 +97,16 @@ public class MapVertexMapper implements InstanceGraphMapper { String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), keyStr); Optional existingEdge = getEdgeIfExists(mapType, currentMap, keyStr); - GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getAttribute(), mapType.getValueType(), entry.getValue()) + GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(), mapType.getValueType(), entry.getValue()) .referringVertex(ctx.getReferringVertex()) .edge(existingEdge) .vertexProperty(propertyNameForKey).build(); + //Add/Update/Remove property value Object newEntry = structVertexMapper.mapCollectionElementsToVertex(mapCtx); + MapVertexMapper.setMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey, newEntry); + newMap.put(keyStr, newEntry); } } @@ -192,7 +195,9 @@ public class MapVertexMapper implements InstanceGraphMapper { private Optional getEdgeIfExists(AtlasMapType mapType, Map currentMap, String keyStr) { Optional existingEdge = Optional.absent(); if ( AtlasGraphUtilsV1.isReference(mapType.getValueType()) ) { - existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr)); + if ( currentMap.get(keyStr) != null) { + existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr)); + } } return existingEdge; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/StructVertexMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/StructVertexMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/StructVertexMapper.java index ae9ecc4..22851ca 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/StructVertexMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/StructVertexMapper.java @@ -17,27 +17,31 @@ */ package org.apache.atlas.repository.store.graph.v1; +import com.google.common.base.Optional; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContextV1; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.RepositoryException; +import org.apache.atlas.repository.graph.AtlasEdgeLabel; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasMapType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + public class StructVertexMapper implements InstanceGraphMapper { private final AtlasGraph graph; @@ -64,18 +68,17 @@ public class StructVertexMapper implements InstanceGraphMapper { @Override public AtlasEdge toGraph(GraphMutationContext ctx) throws AtlasBaseException { - AtlasEdge result = null; - - String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName()); + AtlasEdge ret = null; if ( ctx.getCurrentEdge().isPresent() ) { - updateVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getCurrentEdge().get().getOutVertex()); - result = ctx.getCurrentEdge().get(); - } else { - result = createVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getReferringVertex(), edgeLabel); + updateVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getCurrentEdge().get().getInVertex()); + ret = ctx.getCurrentEdge().get(); + } else if (ctx.getValue() != null) { + String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey()); + ret = createVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getReferringVertex(), edgeLabel); } - return result; + return ret; } @Override @@ -88,44 +91,86 @@ public class StructVertexMapper implements InstanceGraphMapper { /** * Map attributes for entity, struct or trait + * + * @param op * @param structType * @param struct * @param vertex * @return * @throws AtlasBaseException */ - public AtlasVertex mapAttributestoVertex(AtlasStructType structType, AtlasStruct struct, AtlasVertex vertex) throws AtlasBaseException { + public AtlasVertex mapAttributestoVertex(final EntityMutations.EntityOperation op, AtlasStructType structType, AtlasStruct struct, AtlasVertex vertex) throws AtlasBaseException { if (struct.getAttributes() != null) { - for (String attrName : struct.getAttributes().keySet()) { - Object value = struct.getAttribute(attrName); - AtlasType attributeType = structType.getAttributeType(attrName); - if ( attributeType != null) { - final AtlasStructType.AtlasAttribute attribute = structType.getAttribute(attrName); - - GraphMutationContext ctx = new GraphMutationContext.Builder(attribute, value) - .referringVertex(vertex) - .vertexProperty(AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(structType, attrName)).build(); - mapToVertexByTypeCategory(ctx); + if (op.equals(EntityMutations.EntityOperation.CREATE)) { + final Map allAttributes = structType.getAllAttributes(); + for (String attrName : allAttributes.keySet()) { + Object value = struct.getAttribute(attrName); + + mapAttribute(op, structType, attrName, value, vertex); + } + } else if (op.equals(EntityMutations.EntityOperation.UPDATE)) { + for (String attrName : struct.getAttributes().keySet()) { + Object value = struct.getAttribute(attrName); + mapAttribute(op, structType, attrName, value, vertex); } } - - //Set updated timestamp - AtlasGraphUtilsV1.setProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContextV1.get().getRequestTime()); - GraphHelper.setProperty(vertex, Constants.MODIFIED_BY_KEY, RequestContextV1.get().getUser()); + updateModificationMetadata(vertex); } return vertex; } + private void mapAttribute(final EntityMutations.EntityOperation op, AtlasStructType structType, String attrName, Object value, AtlasVertex vertex) throws AtlasBaseException { + AtlasType attributeType = structType.getAttributeType(attrName); + if (attributeType != null) { + final AtlasStructType.AtlasAttribute attribute = structType.getAttribute(attrName); + + if (value == null) { + if ( attribute.getAttributeType().getTypeCategory() == TypeCategory.PRIMITIVE) { + if ( attribute.getAttributeDef().getIsOptional()) { + value = attribute.getAttributeType().createOptionalDefaultValue(); + } else { + value = attribute.getAttributeType().createDefaultValue(); + } + } + } + + final String vertexProperty = structType.getQualifiedAttributeName(attrName); + GraphMutationContext ctx = new GraphMutationContext.Builder(op, attribute, value) + .referringVertex(vertex) + .vertexProperty(GraphHelper.encodePropertyKey(vertexProperty)).build(); + mapToVertexByTypeCategory(ctx); + } + } + + private void updateModificationMetadata(AtlasVertex vertex) { + //Set updated timestamp + AtlasGraphUtilsV1.setProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContextV1.get().getRequestTime()); + GraphHelper.setProperty(vertex, Constants.MODIFIED_BY_KEY, RequestContextV1.get().getUser()); + } + + protected Object mapToVertexByTypeCategory(GraphMutationContext ctx) throws AtlasBaseException { - switch(ctx.getAttrType().getTypeCategory()) { + if (ctx.getOp() == EntityMutations.EntityOperation.CREATE && ctx.getValue() == null) { + return null; + } + + switch (ctx.getAttrType().getTypeCategory()) { case PRIMITIVE: case ENUM: return primitivesToVertex(ctx); case STRUCT: + String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey()); + AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel); + Optional edge = currentEdge != null ? Optional.of(currentEdge) : Optional.absent(); + ctx.setExistingEdge(edge); return toGraph(ctx); case ENTITY: + edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey()); + currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel); AtlasEntityType instanceType = entityVertexMapper.getInstanceType(ctx.getValue()); + edge = currentEdge != null ? Optional.of(currentEdge) : Optional.absent(); ctx.setElementType(instanceType); + ctx.setExistingEdge(edge); return entityVertexMapper.toGraph(ctx); case MAP: return mapVertexMapper.toGraph(ctx); @@ -137,17 +182,13 @@ public class StructVertexMapper implements InstanceGraphMapper { } protected Object primitivesToVertex(GraphMutationContext ctx) { - if ( ctx.getAttrType().getTypeCategory() == TypeCategory.MAP ) { - MapVertexMapper.setMapValueProperty(((AtlasMapType) ctx.getAttrType()).getValueType(), ctx.getReferringVertex(), ctx.getVertexPropertyKey(), ctx.getValue()); - } else { - AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey(), ctx.getValue()); - } + AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey(), ctx.getValue()); return ctx.getValue(); } private AtlasEdge createVertex(AtlasStructType parentType, AtlasStructType attrType, AtlasStructDef.AtlasAttributeDef attributeDef, AtlasStruct struct, AtlasVertex referringVertex, String edgeLabel) throws AtlasBaseException { AtlasVertex vertex = createVertexTemplate(struct, attrType); - mapAttributestoVertex(attrType, struct, vertex); + mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, attrType, struct, vertex); try { //TODO - Map directly in AtlasGraphUtilsV1 @@ -158,7 +199,7 @@ public class StructVertexMapper implements InstanceGraphMapper { } private void updateVertex(AtlasStructType parentType, AtlasStructType structAttributeType, AtlasStructDef.AtlasAttributeDef attributeDef, AtlasStruct value, AtlasVertex structVertex) throws AtlasBaseException { - mapAttributestoVertex(structAttributeType, value, structVertex); + mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, structAttributeType, value, structVertex); } protected AtlasVertex createVertexTemplate(final AtlasStruct instance, final AtlasStructType structType) { @@ -187,7 +228,7 @@ public class StructVertexMapper implements InstanceGraphMapper { switch(ctx.getAttrType().getTypeCategory()) { case PRIMITIVE: case ENUM: - return primitivesToVertex(ctx); + return ctx.getValue(); case STRUCT: return toGraph(ctx); case ENTITY: http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6ccba52c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java index 8025f1e..2698504 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java @@ -20,6 +20,7 @@ package org.apache.atlas.repository.store.graph.v1; import com.google.common.base.Optional; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.typedef.AtlasStructDef; @@ -98,7 +99,12 @@ public class UniqAttrBasedEntityResolver implements EntityResolver { } Optional resolveByUniqueAttribute(AtlasEntity entity) throws AtlasBaseException { - AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName()); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + if (entityType == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName()); + } + for (AtlasStructType.AtlasAttribute attr : entityType.getAllAttributes().values()) { if (attr.getAttributeDef().getIsUnique()) { Object attrVal = entity.getAttribute(attr.getAttributeDef().getName());