Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D22F9200C1A for ; Mon, 13 Feb 2017 08:45:14 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D0B5A160B60; Mon, 13 Feb 2017 07:45:14 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8E730160B4D for ; Mon, 13 Feb 2017 08:45:12 +0100 (CET) Received: (qmail 68105 invoked by uid 500); 13 Feb 2017 07:45:11 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 68096 invoked by uid 99); 13 Feb 2017 07:45:11 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Feb 2017 07:45:11 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 3B1DA1A7ABB for ; Mon, 13 Feb 2017 07:45:11 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.219 X-Spam-Level: X-Spam-Status: No, score=-6.219 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id PhjXByjIyppi for ; Mon, 13 Feb 2017 07:44:59 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 5E72E5FB44 for ; Mon, 13 Feb 2017 07:44:57 +0000 (UTC) Received: (qmail 67609 invoked by uid 99); 13 Feb 2017 07:43:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Feb 2017 07:43:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C2B82DFF88; Mon, 13 Feb 2017 07:43:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@atlas.incubator.apache.org Date: Mon, 13 Feb 2017 07:44:01 -0000 Message-Id: <45fed34f253d43b78d7edb6842aa5c18@git.apache.org> In-Reply-To: <63f26161a855417292d3e48f1c8391bc@git.apache.org> References: <63f26161a855417292d3e48f1c8391bc@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [5/5] incubator-atlas git commit: ATLAS-1544: implementation of REST endpoints for entity create/update/bulk-get archived-at: Mon, 13 Feb 2017 07:45:15 -0000 ATLAS-1544: implementation of REST endpoints for entity create/update/bulk-get Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ce20d6f5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ce20d6f5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ce20d6f5 Branch: refs/heads/master Commit: ce20d6f59da89b245c8e4d393dd87ff62f95847f Parents: 414b7bb Author: Madhan Neethiraj Authored: Fri Feb 10 11:53:59 2017 -0800 Committer: Madhan Neethiraj Committed: Sun Feb 12 23:40:27 2017 -0800 ---------------------------------------------------------------------- .../org/apache/atlas/AtlasEntitiesClientV2.java | 122 ++-- .../java/org/apache/atlas/AtlasErrorCode.java | 5 +- .../atlas/model/instance/AtlasEntity.java | 57 +- .../atlas/model/instance/AtlasObjectId.java | 109 +-- .../atlas/model/instance/AtlasStruct.java | 2 +- .../org/apache/atlas/type/AtlasArrayType.java | 4 +- .../apache/atlas/type/AtlasBuiltInTypes.java | 15 +- .../org/apache/atlas/type/AtlasEntityType.java | 3 + .../org/apache/atlas/type/AtlasStructType.java | 52 +- .../org/apache/atlas/type/AtlasTypeUtil.java | 30 +- .../test/java/org/apache/atlas/TestUtilsV2.java | 154 +++- .../atlas/type/TestAtlasObjectIdType.java | 42 +- .../apache/atlas/RepositoryMetadataModule.java | 18 - .../atlas/repository/graph/GraphHelper.java | 2 +- .../bootstrap/AtlasTypeDefStoreInitializer.java | 72 +- .../store/graph/AtlasEntityStore.java | 81 +-- .../store/graph/EntityGraphDiscovery.java | 4 +- .../graph/EntityGraphDiscoveryContext.java | 157 ++-- .../repository/store/graph/EntityResolver.java | 7 +- .../store/graph/v1/ArrayVertexMapper.java | 178 ----- .../graph/v1/AtlasEntityGraphDiscoveryV1.java | 302 ++++---- .../store/graph/v1/AtlasEntityStoreV1.java | 243 +++---- .../store/graph/v1/AtlasEntityStream.java | 76 ++ .../store/graph/v1/AtlasGraphUtilsV1.java | 2 +- .../graph/v1/AttributeMutationContext.java | 150 ++++ .../store/graph/v1/DeleteHandlerV1.java | 6 +- .../store/graph/v1/EntityGraphMapper.java | 697 +++++++++++++++--- .../store/graph/v1/EntityGraphRetriever.java | 19 +- .../store/graph/v1/EntityMutationContext.java | 45 +- .../repository/store/graph/v1/EntityStream.java | 2 +- .../store/graph/v1/GraphMutationContext.java | 209 ------ .../store/graph/v1/IDBasedEntityResolver.java | 84 +-- .../store/graph/v1/InMemoryMapEntityStream.java | 15 +- .../store/graph/v1/InstanceGraphMapper.java | 9 +- .../store/graph/v1/MapVertexMapper.java | 201 ------ .../store/graph/v1/StructVertexMapper.java | 258 ------- .../graph/v1/UniqAttrBasedEntityResolver.java | 147 +--- .../store/graph/v1/AtlasEntityStoreV1Test.java | 723 +++++++++---------- .../service/DefaultMetadataServiceTest.java | 2 +- .../org/apache/atlas/examples/QuickStartV2.java | 18 +- .../adapters/AtlasEntityFormatConverter.java | 4 +- .../web/adapters/AtlasObjectIdConverter.java | 4 +- .../org/apache/atlas/web/rest/EntitiesREST.java | 104 +-- .../org/apache/atlas/web/rest/EntityREST.java | 189 ++--- .../atlas/web/adapters/TestEntitiesREST.java | 84 ++- .../atlas/web/adapters/TestEntityREST.java | 49 +- .../atlas/web/resources/BaseResourceIT.java | 6 +- .../web/resources/EntityV2JerseyResourceIT.java | 75 +- 48 files changed, 2344 insertions(+), 2493 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java b/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java index be3088f..9ad9c16 100644 --- a/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java +++ b/client/src/main/java/org/apache/atlas/AtlasEntitiesClientV2.java @@ -17,7 +17,6 @@ */ package org.apache.atlas; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,11 +24,10 @@ import javax.ws.rs.HttpMethod; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; -import org.apache.atlas.model.SearchFilter; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications; import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntities; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.commons.collections.MapUtils; @@ -37,34 +35,33 @@ import org.apache.commons.configuration.Configuration; import org.apache.hadoop.security.UserGroupInformation; import com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.core.util.MultivaluedMapImpl; public class AtlasEntitiesClientV2 extends AtlasBaseClient { - - private static final GenericType> ENTITY_LIST_TYPE = new GenericType>(){}; public static final String ENTITY_API = BASE_URI + "v2/entity/"; - public static final String ENTITIES_API = BASE_URI + "v2/entities/"; - - private static final APIInfo GET_ENTITY_BY_GUID = new APIInfo(ENTITY_API + "guid/", HttpMethod.GET, Response.Status.OK); - private static final APIInfo CREATE_ENTITY = new APIInfo(ENTITIES_API, HttpMethod.POST, Response.Status.OK); - private static final APIInfo UPDATE_ENTITY = CREATE_ENTITY; - private static final APIInfo GET_ENTITY_BY_ATTRIBUTE = new APIInfo(ENTITY_API + "uniqueAttribute/type/%s", HttpMethod.GET, Response.Status.OK); - private static final APIInfo UPDATE_ENTITY_BY_ATTRIBUTE = new APIInfo(ENTITY_API + "uniqueAttribute/type/%s/attribute/%s", HttpMethod.PUT, Response.Status.OK); - private static final APIInfo DELETE_ENTITY_BY_ATTRIBUTE = new APIInfo(ENTITY_API + "uniqueAttribute/type/%s/attribute/%s", HttpMethod.DELETE, Response.Status.OK); - private static final APIInfo DELETE_ENTITY_BY_GUID = new APIInfo(ENTITY_API + "guid/", HttpMethod.DELETE, Response.Status.OK); - private static final APIInfo DELETE_ENTITY_BY_GUIDS = new APIInfo(ENTITIES_API + "guids/", HttpMethod.DELETE, Response.Status.OK); - private static final APIInfo GET_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.GET, Response.Status.OK); - private static final APIInfo ADD_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.POST, Response.Status.NO_CONTENT); + private static final APIInfo GET_ENTITY_BY_GUID = new APIInfo(ENTITY_API + "guid/%s", HttpMethod.GET, Response.Status.OK); + private static final APIInfo GET_ENTITY_BY_ATTRIBUTE = new APIInfo(ENTITY_API + "uniqueAttribute/type/%s", HttpMethod.GET, Response.Status.OK); + private static final APIInfo CREATE_ENTITY = new APIInfo(ENTITY_API, HttpMethod.POST, Response.Status.OK); + private static final APIInfo UPDATE_ENTITY = CREATE_ENTITY; + private static final APIInfo UPDATE_ENTITY_BY_ATTRIBUTE = new APIInfo(ENTITY_API + "uniqueAttribute/type/%s", HttpMethod.PUT, Response.Status.OK); + private static final APIInfo DELETE_ENTITY_BY_GUID = new APIInfo(ENTITY_API + "guid/%s", HttpMethod.DELETE, Response.Status.OK); + private static final APIInfo DELETE_ENTITY_BY_ATTRIBUTE = new APIInfo(ENTITY_API + "uniqueAttribute/type/%s", HttpMethod.DELETE, Response.Status.OK); + + private static final APIInfo GET_ENTITIES_BY_GUIDS = new APIInfo(ENTITY_API + "bulk/", HttpMethod.GET, Response.Status.OK); + private static final APIInfo CREATE_ENTITIES = new APIInfo(ENTITY_API + "bulk/", HttpMethod.POST, Response.Status.OK); + private static final APIInfo UPDATE_ENTITIES = CREATE_ENTITIES; + private static final APIInfo DELETE_ENTITIES_BY_GUIDS = new APIInfo(ENTITY_API + "bulk/", HttpMethod.DELETE, Response.Status.OK); + + private static final APIInfo GET_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.GET, Response.Status.OK); + private static final APIInfo ADD_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.POST, Response.Status.NO_CONTENT); private static final APIInfo UPDATE_CLASSIFICATIONS = new APIInfo(ENTITY_API + "guid/%s/classifications", HttpMethod.PUT, Response.Status.OK); - private static final APIInfo DELETE_CLASSIFICATION = new APIInfo(ENTITY_API + "guid/%s/classification/%s", HttpMethod.DELETE, Response.Status.NO_CONTENT); - private static final APIInfo GET_ENTITIES = new APIInfo(ENTITIES_API + "guids/", HttpMethod.GET, Response.Status.OK); - private static final APIInfo CREATE_ENTITIES = new APIInfo(ENTITIES_API, HttpMethod.POST, Response.Status.OK); - private static final APIInfo UPDATE_ENTITIES = CREATE_ENTITIES; - private static final APIInfo DELETE_ENTITIES = new APIInfo(ENTITIES_API + "guids/", HttpMethod.GET, Response.Status.NO_CONTENT); - private static final APIInfo SEARCH_ENTITIES = new APIInfo(ENTITIES_API, HttpMethod.GET, Response.Status.OK); + private static final APIInfo DELETE_CLASSIFICATION = new APIInfo(ENTITY_API + "guid/%s/classification/%s", HttpMethod.DELETE, Response.Status.NO_CONTENT); + + + public static final String PREFIX_ATTR = "attr:"; + public AtlasEntitiesClientV2(String[] baseUrl, String[] basicAuthUserNamePassword) { super(baseUrl, basicAuthUserNamePassword); @@ -88,50 +85,61 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient { } public AtlasEntityWithExtInfo getEntityByGuid(String guid) throws AtlasServiceException { - - return callAPI(GET_ENTITY_BY_GUID, null, AtlasEntityWithExtInfo.class, guid); + return callAPI(formatPathForPathParams(GET_ENTITY_BY_GUID, guid), AtlasEntityWithExtInfo.class, null); } - public AtlasEntities getEntityByGuids(List guids) throws AtlasServiceException { - return callAPI(GET_ENTITY_BY_GUID, AtlasEntities.class, "guid", guids); + public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map attributes) throws AtlasServiceException { + MultivaluedMap queryParams = attributesToQueryParams(attributes); + + return callAPI(formatPathForPathParams(GET_ENTITY_BY_ATTRIBUTE, type), AtlasEntityWithExtInfo.class, queryParams); } - public static final String PREFIX_ATTR = "attr:"; + public EntityMutationResponse updateEntityByAttribute(String type, Map attributes, AtlasEntity entity) throws AtlasServiceException { + MultivaluedMap queryParams = attributesToQueryParams(attributes); - public AtlasEntityWithExtInfo getEntityByAttribute(String type, Map attributes) throws AtlasServiceException { + return callAPI(formatPathForPathParams(UPDATE_ENTITY_BY_ATTRIBUTE, type), entity, EntityMutationResponse.class, queryParams); + } + + public EntityMutationResponse deleteEntityByAttribute(String type, Map attributes) throws AtlasServiceException { MultivaluedMap queryParams = attributesToQueryParams(attributes); - return callAPI(formatPathForPathParams(GET_ENTITY_BY_ATTRIBUTE, type), AtlasEntityWithExtInfo.class, queryParams); + return callAPI(formatPathForPathParams(DELETE_ENTITY_BY_ATTRIBUTE, type), null, EntityMutationResponse.class, queryParams); } - public EntityMutationResponse updateEntityByAttribute(String type, String attribute, String value, AtlasEntity entity) throws AtlasServiceException { - MultivaluedMap queryParams = new MultivaluedMapImpl(); - queryParams.add("value", value); - return callAPI(formatPathForPathParams(UPDATE_ENTITY_BY_ATTRIBUTE, type, attribute), entity, EntityMutationResponse.class, queryParams); + public EntityMutationResponse createEntity(AtlasEntityWithExtInfo entity) throws AtlasServiceException { + return callAPI(CREATE_ENTITY, entity, EntityMutationResponse.class); } - public EntityMutationResponse deleteEntityByAttribute(String type, String attribute, String value) throws AtlasServiceException { - MultivaluedMap queryParams = new MultivaluedMapImpl(); - queryParams.add("value", value); - return callAPI(formatPathForPathParams(DELETE_ENTITY_BY_ATTRIBUTE, type, attribute), null, EntityMutationResponse.class, queryParams); + public EntityMutationResponse updateEntity(AtlasEntityWithExtInfo entity) throws AtlasServiceException { + return callAPI(UPDATE_ENTITY, entity, EntityMutationResponse.class); } - public EntityMutationResponse createEntity(final AtlasEntity atlasEntity) throws AtlasServiceException { - return callAPI(CREATE_ENTITY, new HashMap(1) {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class); + public EntityMutationResponse deleteEntityByGuid(String guid) throws AtlasServiceException { + return callAPI(formatPathForPathParams(DELETE_ENTITY_BY_GUID, guid), EntityMutationResponse.class, null); } - public EntityMutationResponse updateEntity(final AtlasEntity atlasEntity) throws AtlasServiceException { - return callAPI(UPDATE_ENTITY, new HashMap(1) {{ put(atlasEntity.getGuid(), atlasEntity); }}, EntityMutationResponse.class); + + public AtlasEntitiesWithExtInfo getEntitiesByGuids(List guids) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + + queryParams.put("guid", guids); + + return callAPI(GET_ENTITIES_BY_GUIDS, AtlasEntitiesWithExtInfo.class, queryParams); } - public AtlasEntity deleteEntityByGuid(String guid) throws AtlasServiceException { - return callAPI(DELETE_ENTITY_BY_GUID, null, AtlasEntity.class, guid); + public EntityMutationResponse createEntities(AtlasEntitiesWithExtInfo atlasEntities) throws AtlasServiceException { + return callAPI(CREATE_ENTITIES, atlasEntities, EntityMutationResponse.class); } - public EntityMutationResponse deleteEntityByGuid(List guids) throws AtlasServiceException { - return callAPI(DELETE_ENTITY_BY_GUIDS, EntityMutationResponse.class, "guid", guids); + public EntityMutationResponse updateEntities(AtlasEntitiesWithExtInfo atlasEntities) throws AtlasServiceException { + return callAPI(UPDATE_ENTITIES, atlasEntities, EntityMutationResponse.class); } + public EntityMutationResponse deleteEntitiesByGuids(List guids) throws AtlasServiceException { + return callAPI(DELETE_ENTITIES_BY_GUIDS, EntityMutationResponse.class, "guid", guids); + } + + public AtlasClassifications getClassifications(String guid) throws AtlasServiceException { return callAPI(formatPathForPathParams(GET_CLASSIFICATIONS, guid), null, AtlasClassifications.class); } @@ -158,26 +166,6 @@ public class AtlasEntitiesClientV2 extends AtlasBaseClient { return null; } - public EntityMutationResponse createEntities(List atlasEntities) throws AtlasServiceException { - - return callAPI(CREATE_ENTITIES, entityListToMap(atlasEntities), EntityMutationResponse.class); - } - - private Map entityListToMap(List atlasEntities) { - Map toSend = new HashMap(atlasEntities.size()); - for(AtlasEntity entity : atlasEntities) { - toSend.put(entity.getGuid(), entity); - } - return toSend; - } - - public EntityMutationResponse updateEntities(List atlasEntities) throws AtlasServiceException { - return callAPI(UPDATE_ENTITIES, entityListToMap(atlasEntities), EntityMutationResponse.class); - } - - public AtlasEntity.AtlasEntities searchEntities(SearchFilter searchFilter) throws AtlasServiceException { - return callAPI(GET_ENTITIES, AtlasEntity.AtlasEntities.class, searchFilter.getParams()); - } private MultivaluedMap attributesToQueryParams(Map attributes) { return attributesToQueryParams(attributes, null); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 5902463..adffee7 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -64,6 +64,7 @@ public enum AtlasErrorCode { UNRESOLVED_REFERENCES_FOUND(400, "ATLAS40033E", "Unresolved references: byId={0}; byUniqueAttributes={1}"), UNKNOWN_ATTRIBUTE(400, "ATLAS40034E", "Attribute {0} not found for type {1}"), SYSTEM_TYPE(400, "ATLAS40035E", "{0} is a System-type"), + INVALID_STRUCT_VALUE(400, "ATLAS40036E", "not a valid struct value {0}"), // All Not found enums go here TYPE_NAME_NOT_FOUND(404, "ATLAS4041E", "Given typename {0} was invalid"), @@ -74,8 +75,8 @@ public enum AtlasErrorCode { INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"), DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"), INSTANCE_CRUD_INVALID_PARAMS(404, "ATLAS4049E", "Invalid instance creation/updation parameters passed : {0}"), - INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(404, "ATLAS40410E", "Instance {0} with unique attribute {1}={2} does not exist"), - + INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(404, "ATLAS40410E", "Instance {0} with unique attribute {1} does not exist"), + REFERENCED_ENTITY_NOT_FOUND(404, "ATLAS40411E", "Referenced entity {0} is not found"), // All data conflict errors go here TYPE_ALREADY_EXISTS(409, "ATLAS4091E", "Given type {0} already exists"), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index af936cc..edaede0 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -34,6 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -87,16 +88,16 @@ public class AtlasEntity extends AtlasStruct implements Serializable { this(entityDef != null ? entityDef.getName() : null, null); } + public AtlasEntity(String typeName, String attrName, Object attrValue) { + super(typeName, attrName, attrValue); + + init(); + } + public AtlasEntity(String typeName, Map attributes) { super(typeName, attributes); - setGuid(nextInternalId()); - setStatus(null); - setCreatedBy(null); - setUpdatedBy(null); - setCreateTime(null); - setUpdateTime(null); - setClassifications(null); + init(); } public AtlasEntity(AtlasEntity other) { @@ -186,13 +187,15 @@ public class AtlasEntity extends AtlasStruct implements Serializable { @JsonIgnore public static boolean isAssigned(String guid) { - try { - UUID.fromString(guid); - } catch (IllegalArgumentException e) { - return false; + if (guid != null) { + try { + UUID.fromString(guid); + return true; + } catch (IllegalArgumentException e) { + // ignore + } } - - return true; + return false; } @JsonIgnore @@ -200,13 +203,23 @@ public class AtlasEntity extends AtlasStruct implements Serializable { return guid != null && guid.length() > 0 && guid.charAt(0) == '-'; } + private void init() { + setGuid(nextInternalId()); + setStatus(null); + setCreatedBy(null); + setUpdatedBy(null); + setCreateTime(null); + setUpdateTime(null); + setClassifications(null); + } + private static String nextInternalId() { return "-" + Long.toString(s_nextId.getAndIncrement()); } @JsonIgnore public AtlasObjectId getAtlasObjectId() { - return new AtlasObjectId(getTypeName(), getGuid()); + return new AtlasObjectId(getGuid(), getTypeName()); } @Override @@ -279,6 +292,10 @@ public class AtlasEntity extends AtlasStruct implements Serializable { setReferredEntities(null); } + public AtlasEntityExtInfo(AtlasEntity referredEntity) { + addReferredEntity(referredEntity); + } + public AtlasEntityExtInfo(Map referredEntities) { setReferredEntities(referredEntities); } @@ -294,6 +311,11 @@ public class AtlasEntity extends AtlasStruct implements Serializable { public void setReferredEntities(Map referredEntities) { this.referredEntities = referredEntities; } @JsonIgnore + public final void addReferredEntity(AtlasEntity entity) { + addReferredEntity(entity.getGuid(), entity); + } + + @JsonIgnore public final void addReferredEntity(String guid, AtlasEntity entity) { Map r = this.referredEntities; @@ -494,10 +516,17 @@ public class AtlasEntity extends AtlasStruct implements Serializable { this(null, null); } + public AtlasEntitiesWithExtInfo(AtlasEntity entity) { this(Arrays.asList(entity), null); + } + public AtlasEntitiesWithExtInfo(List entities) { this(entities, null); } + public AtlasEntitiesWithExtInfo(AtlasEntityWithExtInfo entity) { + this(Arrays.asList(entity.getEntity()), entity); + } + public AtlasEntitiesWithExtInfo(List entities, AtlasEntityExtInfo extInfo) { super(extInfo); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java index e4abda7..dea3a93 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java @@ -20,6 +20,7 @@ package org.apache.atlas.model.instance; import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.HashMap; import java.util.Objects; import javax.xml.bind.annotation.XmlAccessType; @@ -51,88 +52,83 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; public class AtlasObjectId implements Serializable { private static final long serialVersionUID = 1L; - public static final String KEY_TYPENAME = "typeName"; public static final String KEY_GUID = "guid"; + public static final String KEY_TYPENAME = "typeName"; public static final String KEY_UNIQUE_ATTRIBUTES = "uniqueAttributes"; - private String typeName; private String guid; + private String typeName; private Map uniqueAttributes; - @JsonIgnore - private boolean isAssignedGuid = false; - - @JsonIgnore - private boolean isUnAssignedGuid = false; public AtlasObjectId() { - this(null, null, null); + this(null, null, (Map)null); } - public AtlasObjectId(String typeName) { - this(typeName, null, null); + public AtlasObjectId(String guid) { + this(guid, null, (Map)null); } - public AtlasObjectId(String typeName, String guid) { - this(typeName, guid, null); + public AtlasObjectId(String guid, String typeName) { + this(guid, typeName, (Map)null); } public AtlasObjectId(String typeName, Map uniqueAttributes) { - this(typeName, null, uniqueAttributes); + this(null, typeName, uniqueAttributes); } - public AtlasObjectId(String typeName, String guid, Map uniqueAttributes) { - setTypeName(typeName); + public AtlasObjectId(String typeName, final String attrName, final Object attrValue) { + this(null, typeName, new HashMap() {{ put(attrName, attrValue); }}); + } + + public AtlasObjectId(String guid, String typeName, Map uniqueAttributes) { setGuid(guid); + setTypeName(typeName); setUniqueAttributes(uniqueAttributes); } public AtlasObjectId(AtlasObjectId other) { if (other != null) { - setTypeName(other.getTypeName()); setGuid(other.getGuid()); + setTypeName(other.getTypeName()); setUniqueAttributes(other.getUniqueAttributes()); } } public AtlasObjectId(Map objIdMap) { if (objIdMap != null) { - Object t = objIdMap.get(KEY_TYPENAME); Object g = objIdMap.get(KEY_GUID); + Object t = objIdMap.get(KEY_TYPENAME); Object u = objIdMap.get(KEY_UNIQUE_ATTRIBUTES); - if (t != null) { - setTypeName(t.toString()); - } - if (g != null) { setGuid(g.toString()); } + if (t != null) { + setTypeName(t.toString()); + } + if (u != null && u instanceof Map) { setUniqueAttributes((Map)u); } } } - public String getTypeName() { - return typeName; - } - - public void setTypeName(String typeName) { - this.typeName = typeName; - } - public String getGuid() { return guid; } public void setGuid(String guid) { this.guid = guid; - if ( guid != null) { - this.isAssignedGuid = AtlasEntity.isAssigned(guid); - this.isUnAssignedGuid = AtlasEntity.isUnAssigned(guid); - } + } + + public String getTypeName() { + return typeName; + } + + public void setTypeName(String typeName) { + this.typeName = typeName; } public Map getUniqueAttributes() { @@ -144,28 +140,29 @@ public class AtlasObjectId implements Serializable { } @JsonIgnore + public boolean isValidGuid() { + return isAssignedGuid() || isUnAssignedGuid(); + } + + @JsonIgnore public boolean isAssignedGuid() { - return isAssignedGuid; + return AtlasEntity.isAssigned(guid); } @JsonIgnore public boolean isUnAssignedGuid() { - return isUnAssignedGuid; + return AtlasEntity.isUnAssigned(guid); } @JsonIgnore public boolean isValid() { - if (StringUtils.isEmpty(typeName)) { - return false; - } else if (StringUtils.isNotEmpty(guid)) { - if (!isAssignedGuid && !isUnAssignedGuid) { - return false; - } - } else if (MapUtils.isEmpty(uniqueAttributes)) { - return false; + if (isAssignedGuid() || isUnAssignedGuid()) { + return true; + } else if (StringUtils.isNotEmpty(typeName) && MapUtils.isNotEmpty(uniqueAttributes)) { + return true; } - return true; + return false; } public StringBuilder toString(StringBuilder sb) { @@ -174,8 +171,8 @@ public class AtlasObjectId implements Serializable { } sb.append("AtlasObjectId{"); - sb.append("typeName='").append(typeName).append('\''); - sb.append(", guid='").append(guid).append('\''); + sb.append("guid='").append(guid).append('\''); + sb.append(", typeName='").append(typeName).append('\''); sb.append(", uniqueAttributes={"); AtlasBaseTypeDef.dumpObjects(uniqueAttributes, sb); sb.append('}'); @@ -186,17 +183,27 @@ public class AtlasObjectId implements Serializable { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + AtlasObjectId that = (AtlasObjectId) o; + + if (isValidGuid() && Objects.equals(guid, that.guid)) { + return true; + } + return Objects.equals(typeName, that.typeName) && - Objects.equals(guid, that.guid) && - Objects.equals(uniqueAttributes, that.uniqueAttributes); + Objects.equals(uniqueAttributes, that.uniqueAttributes); } @Override public int hashCode() { - return Objects.hash(typeName, guid, uniqueAttributes); + return isValidGuid() ? Objects.hash(guid) : Objects.hash(typeName, uniqueAttributes); } @Override http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java index c23f4d5..4e3c795 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java @@ -81,7 +81,7 @@ public class AtlasStruct implements Serializable { public AtlasStruct(AtlasStruct other) { if (other != null) { setTypeName(other.getTypeName()); - setAttributes(other.getAttributes()); + setAttributes(new HashMap<>(other.getAttributes())); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java index 02851c4..77f5d80 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java @@ -233,7 +233,7 @@ public class AtlasArrayType extends AtlasType { ret = false; messages.add(objName + ": incorrect number of values. found=" + objList.size() - + "; expected: minCount=" + minCount + ", maxCount=" + maxCount); + + "; expected: minCount=" + minCount + ", maxCount=" + maxCount); } int idx = 0; @@ -248,7 +248,7 @@ public class AtlasArrayType extends AtlasType { ret = false; messages.add(objName + ": incorrect number of values. found=" + arrayLen - + "; expected: minCount=" + minCount + ", maxCount=" + maxCount); + + "; expected: minCount=" + minCount + ", maxCount=" + maxCount); } for (int i = 0; i < arrayLen; i++) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java index 6d24d7e..1bfdf77 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java @@ -519,7 +519,7 @@ public class AtlasBuiltInTypes { @Override public AtlasObjectId createDefaultValue() { - return new AtlasObjectId(AtlasBaseTypeDef.ATLAS_TYPE_ASSET, "test"); + return new AtlasObjectId("-1", AtlasBaseTypeDef.ATLAS_TYPE_ASSET); } @Override @@ -551,13 +551,16 @@ public class AtlasBuiltInTypes { } private boolean isValidMap(Map map) { - if (map.containsKey(AtlasObjectId.KEY_TYPENAME)) { - if (map.containsKey(AtlasObjectId.KEY_GUID)) { - return true; - } else { + Object guid = map.get(AtlasObjectId.KEY_GUID); + + if (guid != null && StringUtils.isNotEmpty(guid.toString())) { + return true; + } else { + Object typeName = map.get(AtlasObjectId.KEY_TYPENAME); + if (typeName != null && StringUtils.isNotEmpty(typeName.toString())) { Object uniqueAttributes = map.get(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES); - if (uniqueAttributes instanceof Map && MapUtils.isNotEmpty((Map)uniqueAttributes)) { + if (uniqueAttributes instanceof Map && MapUtils.isNotEmpty((Map) uniqueAttributes)) { return true; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 7615057..8214095 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -190,6 +190,9 @@ public class AtlasEntityType extends AtlasStructType { if (obj instanceof AtlasObjectId) { AtlasObjectId objId = (AtlasObjectId ) obj; return isAssignableFrom(objId); + } else if (obj instanceof Map) { + AtlasObjectId objId = new AtlasObjectId((Map)obj); + return isAssignableFrom(objId); } for (AtlasEntityType superType : superTypes) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 4b23ecf..232356a 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -254,7 +254,7 @@ public class AtlasStructType extends AtlasType { } } } else if (obj instanceof Map) { - Map map = (Map) obj; + Map attributes = AtlasTypeUtil.toStructAttributes((Map)obj); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { String attrName = attributeDef.getName(); @@ -262,7 +262,7 @@ public class AtlasStructType extends AtlasType { if (attribute != null) { AtlasType dataType = attribute.getAttributeType(); - Object value = map.get(attrName); + Object value = attributes.get(attrName); String fieldName = objName + "." + attrName; if (value != null) { @@ -408,14 +408,16 @@ public class AtlasStructType extends AtlasType { private final AtlasType attributeType; private final AtlasAttributeDef attributeDef; private final String qualifiedName; + private final String vertexPropertyName; private final boolean isOwnedRef; private final String inverseRefAttribute; public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) { - this.definedInType = definedInType; - this.attributeDef = attrDef; - this.attributeType = attributeType; - this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName()); + this.definedInType = definedInType; + this.attributeDef = attrDef; + this.attributeType = attributeType; + this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName()); + this.vertexPropertyName = encodePropertyKey(this.qualifiedName); boolean isOwnedRef = false; String inverseRefAttribute = null; @@ -458,17 +460,47 @@ public class AtlasStructType extends AtlasType { public String getQualifiedName() { return qualifiedName; } - public String getQualifiedAttributeName() { - return qualifiedName; - } + public String getVertexPropertyName() { return vertexPropertyName; } public boolean isOwnedRef() { return isOwnedRef; } public String getInverseRefAttribute() { return inverseRefAttribute; } - public static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) { + private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) { final String typeName = structDef.getName(); return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName); } + + private static String encodePropertyKey(String key) { + if (StringUtils.isBlank(key)) { + return key; + } + + for (String[] strMap : RESERVED_CHAR_ENCODE_MAP) { + key = key.replace(strMap[0], strMap[1]); + } + + return key; + } + + private static String decodePropertyKey(String key) { + if (StringUtils.isBlank(key)) { + return key; + } + + for (String[] strMap : RESERVED_CHAR_ENCODE_MAP) { + key = key.replace(strMap[1], strMap[0]); + } + + return key; + } + + private static String[][] RESERVED_CHAR_ENCODE_MAP = new String[][] { + new String[] { "{", "_o" }, + new String[] { "}", "_c" }, + new String[] { "\"", "_q" }, + new String[] { "$", "_d" }, + new String[] { "%", "_p" }, + }; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java index 3ba7a33..0a82f40 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java @@ -21,6 +21,8 @@ import com.google.common.collect.ImmutableSet; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; @@ -35,14 +37,16 @@ import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; - import java.util.Map; import java.util.Set; -import java.util.Arrays; + import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -298,4 +302,26 @@ public class AtlasTypeUtil { return headerList; } + + public static Collection toObjectIds(Collection entities) { + List ret = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(entities)) { + for (AtlasEntity entity : entities) { + if (entity != null) { + ret.add(entity.getAtlasObjectId()); + } + } + } + + return ret; + } + + public static Map toStructAttributes(Map map) { + if (map != null && map.containsKey("typeName") && map.containsKey("attributes") && map.get("attributes") instanceof Map) { + return (Map)map.get("attributes"); + } + + return map; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/test/java/org/apache/atlas/TestUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index f09a4b9..f268e48 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -21,6 +21,8 @@ package org.apache.atlas; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.typedef.AtlasClassificationDef; @@ -275,8 +277,98 @@ public final class TestUtilsV2 { } public static final String DEPARTMENT_TYPE = "Department"; - public static final String PERSON_TYPE = "Person"; - public static final String EMPLOYEE_TYPE = "Employee"; + public static final String EMPLOYEE_TYPE = "Employee"; + public static final String MANAGER_TYPE = "Manager"; + public static final String ADDRESS_TYPE = "Address"; + + public static AtlasEntitiesWithExtInfo createDeptEg2() { + AtlasEntitiesWithExtInfo entitiesWithExtInfo = new AtlasEntitiesWithExtInfo(); + + /******* Department - HR *******/ + AtlasEntity hrDept = new AtlasEntity(DEPARTMENT_TYPE, "name", "hr"); + AtlasObjectId hrDeptId = hrDept.getAtlasObjectId(); + + /******* Address Entities *******/ + AtlasStruct janeAddr = new AtlasStruct(ADDRESS_TYPE); + janeAddr.setAttribute("street", "Great America Parkway"); + janeAddr.setAttribute("city", "Santa Clara"); + + AtlasStruct juliusAddr = new AtlasStruct(ADDRESS_TYPE); + juliusAddr.setAttribute("street", "Madison Ave"); + juliusAddr.setAttribute("city", "Newtonville"); + + AtlasStruct maxAddr = new AtlasStruct(ADDRESS_TYPE); + maxAddr.setAttribute("street", "Ripley St"); + maxAddr.setAttribute("city", "Newton"); + + AtlasStruct johnAddr = new AtlasStruct(ADDRESS_TYPE); + johnAddr.setAttribute("street", "Stewart Drive"); + johnAddr.setAttribute("city", "Sunnyvale"); + + /******* Manager - Jane (John and Max subordinates) *******/ + AtlasEntity jane = new AtlasEntity(MANAGER_TYPE); + AtlasObjectId janeId = jane.getAtlasObjectId(); + jane.setAttribute("name", "Jane"); + jane.setAttribute("department", hrDeptId); + jane.setAttribute("address", janeAddr); + + /******* Manager - Julius (no subordinates) *******/ + AtlasEntity julius = new AtlasEntity(MANAGER_TYPE); + AtlasObjectId juliusId = julius.getAtlasObjectId(); + julius.setAttribute("name", "Julius"); + julius.setAttribute("department", hrDeptId); + julius.setAttribute("address", juliusAddr); + julius.setAttribute("subordinates", ImmutableList.of()); + + /******* Employee - Max (Manager: Jane, Mentor: Julius) *******/ + AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE); + AtlasObjectId maxId = max.getAtlasObjectId(); + max.setAttribute("name", "Max"); + max.setAttribute("department", hrDeptId); + max.setAttribute("address", maxAddr); + max.setAttribute("manager", janeId); + max.setAttribute("mentor", juliusId); + max.setAttribute("birthday",new Date(1979, 3, 15)); + max.setAttribute("hasPets", true); + max.setAttribute("age", 36); + max.setAttribute("numberOfCars", 2); + max.setAttribute("houseNumber", 17); + max.setAttribute("carMileage", 13); + max.setAttribute("shares", Long.MAX_VALUE); + max.setAttribute("salary", Double.MAX_VALUE); + max.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000")); + max.setAttribute("approximationOfPi", new BigDecimal("3.1415926535897932")); + + /******* Employee - John (Manager: Jane, Mentor: Max) *******/ + AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE); + AtlasObjectId johnId = john.getAtlasObjectId(); + john.setAttribute("name", "John"); + john.setAttribute("department", hrDeptId); + john.setAttribute("address", johnAddr); + john.setAttribute("manager", janeId); + john.setAttribute("mentor", maxId); + john.setAttribute("birthday",new Date(1950, 5, 15)); + john.setAttribute("hasPets", true); + john.setAttribute("numberOfCars", 1); + john.setAttribute("houseNumber", 153); + john.setAttribute("carMileage", 13364); + john.setAttribute("shares", 15000); + john.setAttribute("salary", 123345.678); + john.setAttribute("age", 50); + john.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000")); + john.setAttribute("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); + + jane.setAttribute("subordinates", ImmutableList.of(johnId, maxId)); + hrDept.setAttribute("employees", ImmutableList.of(janeId, juliusId, maxId, johnId)); + + entitiesWithExtInfo.addEntity(hrDept); + entitiesWithExtInfo.addEntity(jane); + entitiesWithExtInfo.addEntity(julius); + entitiesWithExtInfo.addEntity(max); + entitiesWithExtInfo.addEntity(john); + + return entitiesWithExtInfo; + } public static Map createDeptEg1() { Map deptEmpEntities = new HashMap<>(); @@ -292,7 +384,7 @@ public final class TestUtilsV2 { AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE); AtlasEntity maxAddr = new AtlasEntity("Address"); - AtlasObjectId deptId = new AtlasObjectId(hrDept.getTypeName(), hrDept.getGuid()); + AtlasObjectId deptId = new AtlasObjectId(hrDept.getGuid(), hrDept.getTypeName()); hrDept.setAttribute("name", "hr"); john.setAttribute("name", "John"); john.setAttribute("department", deptId); @@ -329,8 +421,8 @@ public final class TestUtilsV2 { AtlasObjectId johnId = john.getAtlasObjectId(); //TODO - Change to MANAGER_TYPE for JULIUS - AtlasObjectId maxId = new AtlasObjectId(EMPLOYEE_TYPE, max.getGuid()); - AtlasObjectId juliusId = new AtlasObjectId(EMPLOYEE_TYPE, julius.getGuid()); + AtlasObjectId maxId = new AtlasObjectId(max.getGuid(), EMPLOYEE_TYPE); + AtlasObjectId juliusId = new AtlasObjectId(julius.getGuid(), EMPLOYEE_TYPE); max.setAttribute("name", "Max"); max.setAttribute("department", deptId); @@ -356,10 +448,6 @@ public final class TestUtilsV2 { jane.setAttribute("subordinates", ImmutableList.of(johnId, maxId)); -// Map secClearanceLevelMap = new HashMap<>(); -// secClearanceLevelMap.put("level", 1); -// jane.setAttribute("SecurityClearance", secClearanceLevelMap); - deptEmpEntities.put(jane.getGuid(), jane); deptEmpEntities.put(john.getGuid(), john); deptEmpEntities.put(julius.getGuid(), julius); @@ -659,27 +747,32 @@ public final class TestUtilsV2 { return RandomStringUtils.randomAlphanumeric(10); } - public static Map createDBEntity() { - Map ret = new HashMap<>(); + public static AtlasEntity createDBEntity() { AtlasEntity entity = new AtlasEntity(DATABASE_TYPE); String dbName = RandomStringUtils.randomAlphanumeric(10); entity.setAttribute(NAME, dbName); entity.setAttribute("description", "us db"); - ret.put(entity.getGuid(), entity); - return ret; + return entity; } - public static Map createTableEntity(String dbId) { - Map ret = new HashMap<>(); + public static AtlasEntityWithExtInfo createDBEntityV2() { + AtlasEntity dbEntity = new AtlasEntity(DATABASE_TYPE); + + dbEntity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10)); + dbEntity.setAttribute("description", "us db"); + + return new AtlasEntityWithExtInfo(dbEntity); + } + public static AtlasEntity createTableEntity(AtlasEntity dbEntity) { AtlasEntity entity = new AtlasEntity(TABLE_TYPE); String tableName = RandomStringUtils.randomAlphanumeric(10); entity.setAttribute(NAME, tableName); entity.setAttribute("description", "random table"); entity.setAttribute("type", "type"); entity.setAttribute("tableType", "MANAGED"); - entity.setAttribute("database", new AtlasObjectId(DATABASE_TYPE, dbId)); + entity.setAttribute("database", dbEntity.getAtlasObjectId()); entity.setAttribute("created", new Date()); Map partAttributes = new HashMap() {{ @@ -692,16 +785,39 @@ public final class TestUtilsV2 { put("key1", "value1"); }}); - ret.put(entity.getGuid(), entity); + return entity; + } + + public static AtlasEntityWithExtInfo createTableEntityV2(AtlasEntity dbEntity) { + AtlasEntity tblEntity = new AtlasEntity(TABLE_TYPE); + + tblEntity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10)); + tblEntity.setAttribute("description", "random table"); + tblEntity.setAttribute("type", "type"); + tblEntity.setAttribute("tableType", "MANAGED"); + tblEntity.setAttribute("database", dbEntity.getAtlasObjectId()); + tblEntity.setAttribute("created", new Date()); + + final AtlasStruct partitionStruct = new AtlasStruct("partition_struct_type", "name", "part0"); + + tblEntity.setAttribute("partitions", new ArrayList() {{ add(partitionStruct); }}); + tblEntity.setAttribute("parametersMap", + new java.util.HashMap() {{ put("key1", "value1"); }}); + + + AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(tblEntity); + + ret.addReferredEntity(dbEntity); + return ret; } - public static AtlasEntity createColumnEntity(String tableId) { + public static AtlasEntity createColumnEntity(AtlasEntity tableEntity) { AtlasEntity entity = new AtlasEntity(COLUMN_TYPE); entity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10)); entity.setAttribute("type", "VARCHAR(32)"); - entity.setAttribute("table", new AtlasObjectId(TABLE_TYPE, tableId)); + entity.setAttribute("table", tableEntity.getAtlasObjectId()); return entity; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/intg/src/test/java/org/apache/atlas/type/TestAtlasObjectIdType.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasObjectIdType.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasObjectIdType.java index 2a3ba55..fb4a183 100644 --- a/intg/src/test/java/org/apache/atlas/type/TestAtlasObjectIdType.java +++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasObjectIdType.java @@ -45,37 +45,41 @@ public class TestAtlasObjectIdType { Map invalidObj2 = new HashMap<>(); Map invalidObj3 = new HashMap<>(); Map invalidObj4 = new HashMap<>(); - Map invalidObj5 = new HashMap<>(); - - validObj1.put(AtlasObjectId.KEY_TYPENAME, "testType"); - validObj1.put(AtlasObjectId.KEY_GUID, "guid-1234"); - validObj2.put(AtlasObjectId.KEY_TYPENAME, "testType"); - validObj2.put(AtlasObjectId.KEY_GUID, 1234); Map uniqAttribs = new HashMap(); uniqAttribs.put("name", "testTypeInstance-1"); + // guid + validObj1.put(AtlasObjectId.KEY_GUID, "guid-1234"); + + // typeName & unique-attributes + validObj2.put(AtlasObjectId.KEY_TYPENAME, "testType"); + validObj2.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, uniqAttribs); + + // guid, typeName & unique-attributes + validObj3.put(AtlasObjectId.KEY_GUID, "guid-1234"); validObj3.put(AtlasObjectId.KEY_TYPENAME, "testType"); validObj3.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, uniqAttribs); - invalidObj1.put(AtlasObjectId.KEY_TYPENAME, "testType"); // no guid - - invalidObj2.put(AtlasObjectId.KEY_GUID, "guid-1234"); // no typeName or uniqueAttribute - invalidObj2.put(AtlasObjectId.KEY_TYPENAME + "-invalid", "testType"); + // no guid or typeName/unique-attributes + invalidObj1.put(AtlasObjectId.KEY_GUID + "-invalid", "guid-1234"); // no guid or typename or uniqueAttribute - invalidObj3.put(AtlasObjectId.KEY_GUID + "-invalid", "guid-1234"); // no guid or typename or uniqueAttribute + // no unique-attributes + invalidObj2.put(AtlasObjectId.KEY_TYPENAME, "testType"); // no guid - invalidObj4.put(AtlasObjectId.KEY_TYPENAME, "testType"); // empty uniqueAttribute - invalidObj4.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new HashMap()); + // empty uniqueAttribute + invalidObj3.put(AtlasObjectId.KEY_TYPENAME, "testType"); + invalidObj3.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new HashMap()); - invalidObj5.put(AtlasObjectId.KEY_TYPENAME, "testType"); // non-map uniqueAttribute - invalidObj5.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new ArrayList()); + // non-map uniqueAttribute + invalidObj4.put(AtlasObjectId.KEY_TYPENAME, "testType"); + invalidObj4.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new ArrayList()); validValues = new Object[] { - null, validObj1, validObj2, validObj3, new AtlasObjectId(), new AtlasObjectId("testType", "guid-1234"), }; + null, validObj1, validObj2, validObj3, new AtlasObjectId(), new AtlasObjectId("guid-1234", "testType"), }; invalidValues = new Object[] { - invalidObj1, invalidObj2, invalidObj3, invalidObj4, invalidObj5, + invalidObj1, invalidObj2, invalidObj3, invalidObj4, Byte.valueOf((byte)1), Short.valueOf((short)1), Integer.valueOf(1), Long.valueOf(1L), Float.valueOf(1), Double.valueOf(1), BigInteger.valueOf(1), BigDecimal.valueOf(1), "1", "", "12ab", "abcd", "-12ab", @@ -116,12 +120,12 @@ public class TestAtlasObjectIdType { if (value instanceof AtlasObjectId) { assertEquals(normalizedValue, value, "value=" + value); } else if (value instanceof Map) { - assertEquals(normalizedValue.getTypeName(), ((Map)value).get(AtlasObjectId.KEY_TYPENAME).toString(), + assertEquals(normalizedValue.getTypeName(), ((Map)value).get(AtlasObjectId.KEY_TYPENAME), "value=" + value); if (((Map)value).get(AtlasObjectId.KEY_GUID) == null) { assertEquals(normalizedValue.getGuid(), ((Map)value).get(AtlasObjectId.KEY_GUID), "value=" + value); } else { - assertEquals(normalizedValue.getGuid().toString(), ((Map) value).get(AtlasObjectId.KEY_GUID).toString(), "value=" + value); + assertEquals(normalizedValue.getGuid(), ((Map) value).get(AtlasObjectId.KEY_GUID), "value=" + value); } assertEquals(normalizedValue.getUniqueAttributes(), ((Map)value).get(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES), http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index c4d5020..ce29f8d 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -42,17 +42,10 @@ import org.apache.atlas.repository.graph.DeleteHandler; import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.EntityGraphDiscovery; -import org.apache.atlas.repository.store.graph.EntityResolver; -import org.apache.atlas.repository.store.graph.v1.ArrayVertexMapper; -import org.apache.atlas.repository.store.graph.v1.AtlasEntityGraphDiscoveryV1; import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1; import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1; import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper; -import org.apache.atlas.repository.store.graph.v1.IDBasedEntityResolver; -import org.apache.atlas.repository.store.graph.v1.MapVertexMapper; -import org.apache.atlas.repository.store.graph.v1.UniqAttrBasedEntityResolver; import org.apache.atlas.repository.typestore.GraphBackedTypeStore; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.service.Service; @@ -121,15 +114,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { bind(EntityGraphMapper.class); - bind(MapVertexMapper.class).asEagerSingleton(); - - bind(ArrayVertexMapper.class).asEagerSingleton(); - - Multibinder entityRefResolver = - Multibinder.newSetBinder(binder(), EntityResolver.class); - entityRefResolver.addBinding().to(IDBasedEntityResolver.class); - entityRefResolver.addBinding().to(UniqAttrBasedEntityResolver.class); - //Add EntityAuditListener as EntityChangeListener Multibinder entityChangeListenerBinder = Multibinder.newSetBinder(binder(), EntityChangeListener.class); @@ -138,8 +122,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { MethodInterceptor interceptor = new GraphTransactionInterceptor(); requestInjection(interceptor); bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor); - - bind(EntityGraphDiscovery.class).to(AtlasEntityGraphDiscoveryV1.class); } protected Configuration getConfiguration() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 65de1e5..cbc91f5 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -244,7 +244,7 @@ public final class GraphHelper { /** Iterable edges = outVertex.getEdges(Direction.OUT, edgeLabel); for (AtlasEdge edge : edges) { - if (edge.getId().toString().equals(edgeId)) { + if (edge.getObjectId().toString().equals(edgeId)) { return edge; } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java index fb2c939..fa4a480 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java @@ -86,39 +86,7 @@ public class AtlasTypeDefStoreInitializer { continue; } - AtlasTypesDef typesToCreate = new AtlasTypesDef(); - - if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { - for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { - if (!typeRegistry.isRegisteredType(enumDef.getName())) { - typesToCreate.getEnumDefs().add(enumDef); - } - } - } - - if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) { - for (AtlasStructDef structDef : typesDef.getStructDefs()) { - if (!typeRegistry.isRegisteredType(structDef.getName())) { - typesToCreate.getStructDefs().add(structDef); - } - } - } - - if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) { - for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) { - if (!typeRegistry.isRegisteredType(classificationDef.getName())) { - typesToCreate.getClassificationDefs().add(classificationDef); - } - } - } - - if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) { - for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) { - if (!typeRegistry.isRegisteredType(entityDef.getName())) { - typesToCreate.getEntityDefs().add(entityDef); - } - } - } + AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, typeRegistry); if (typesToCreate.isEmpty()) { LOG.info("No new type in file {}", typeDefFile.getAbsolutePath()); @@ -137,6 +105,44 @@ public class AtlasTypeDefStoreInitializer { applyTypePatches(typeDefStore, typeRegistry, typesDirName); } + public static AtlasTypesDef getTypesToCreate(AtlasTypesDef typesDef, AtlasTypeRegistry typeRegistry) { + AtlasTypesDef typesToCreate = new AtlasTypesDef(); + + if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) { + for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { + if (!typeRegistry.isRegisteredType(enumDef.getName())) { + typesToCreate.getEnumDefs().add(enumDef); + } + } + } + + if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) { + for (AtlasStructDef structDef : typesDef.getStructDefs()) { + if (!typeRegistry.isRegisteredType(structDef.getName())) { + typesToCreate.getStructDefs().add(structDef); + } + } + } + + if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) { + for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) { + if (!typeRegistry.isRegisteredType(classificationDef.getName())) { + typesToCreate.getClassificationDefs().add(classificationDef); + } + } + } + + if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) { + for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) { + if (!typeRegistry.isRegisteredType(entityDef.getName())) { + typesToCreate.getEntityDefs().add(entityDef); + } + } + } + + return typesToCreate; + } + private void applyTypePatches(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) { String typePatchesDirName = typesDirName + File.separator + "patches"; File typePatchesDir = new File(typePatchesDirName); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 4cfc012..f3e9563 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -24,8 +24,8 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.repository.store.graph.v1.EntityStream; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasTypeRegistry; import java.util.List; import java.util.Map; @@ -34,13 +34,6 @@ import java.util.Map; * Persistence/Retrieval API for AtlasEntity */ public interface AtlasEntityStore { - - /** - * Initialization - */ - void init(AtlasTypeRegistry typeRegistry) throws AtlasBaseException; - - /** * * Get entity definition by its guid @@ -50,22 +43,6 @@ public interface AtlasEntityStore { AtlasEntityWithExtInfo getById(String guid) throws AtlasBaseException; /** - * Delete an entity by its guid - * @param guid - * @return - */ - EntityMutationResponse deleteById(String guid); - - /** - * Create or update entities - * @param entities Map of the entity Id(guid or transient Id) to AtlasEntity objects that need to be created - * @return EntityMutationResponse Entity mutations operations with the correspomding set of entities on which these operations were performed - * @throws AtlasBaseException - */ - - EntityMutationResponse createOrUpdate(Map entities) throws AtlasBaseException; - - /** * Batch GET to retrieve entities by their ID * @param guid * @return @@ -73,43 +50,57 @@ public interface AtlasEntityStore { */ AtlasEntitiesWithExtInfo getByIds(List guid) throws AtlasBaseException; - /* - * Return list of deleted entity guids - */ - EntityMutationResponse deleteByIds(List guid) throws AtlasBaseException; - /** * * Get an eneity by its unique attribute - * @param entityType - * @param uniqAttributes - * @return AtlasEntity + * @param entityType type of the entity + * @param uniqAttributes Attributes that uniquely identify the entity + * @return EntityMutationResponse details of the updates performed by this call */ - AtlasEntityWithExtInfo getByUniqueAttribute(AtlasEntityType entityType, Map uniqAttributes) - throws AtlasBaseException; + AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes) + throws AtlasBaseException; + + /** + * Create or update entities in the stream + * @param entityStream AtlasEntityStream + * @return EntityMutationResponse Entity mutations operations with the corresponding set of entities on which these operations were performed + * @throws AtlasBaseException + */ + EntityMutationResponse createOrUpdate(EntityStream entityStream) throws AtlasBaseException; /** * @deprecated - * Create or update a single entity - * @param typeName The entity's type - * @param attributeName Attribute that uniquely identifies the entity - * @param attributeValue The unqiue attribute's value - * @return EntityMutationResponse Entity mutations operations with the correspomding set of entities on which these operations were performed + * Update a single entity + * @param entityType type of the entity + * @param uniqAttributes Attributes that uniquely identify the entity + * @return EntityMutationResponse details of the updates performed by this call * @throws AtlasBaseException * */ + EntityMutationResponse updateByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes, + AtlasEntity entity) throws AtlasBaseException; - EntityMutationResponse updateByUniqueAttribute(String typeName, String attributeName, String attributeValue, AtlasEntity entity) throws AtlasBaseException; + /** + * Delete an entity by its guid + * @param guid + * @return + */ + EntityMutationResponse deleteById(String guid) throws AtlasBaseException; /** * @deprecated - * @param typeName - * @param attributeName - * @param attributeValue - * @return + * @param entityType type of the entity + * @param uniqAttributes Attributes that uniquely identify the entity + * @return EntityMutationResponse details of the updates performed by this call * @throws AtlasBaseException */ - EntityMutationResponse deleteByUniqueAttribute(String typeName, String attributeName, String attributeValue) throws AtlasBaseException; + EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityType, Map uniqAttributes) + throws AtlasBaseException; + + /* + * Return list of deleted entity guids + */ + EntityMutationResponse deleteByIds(List guid) throws AtlasBaseException; /** * Add classification(s) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java index 38fca03..a12ba3b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java @@ -20,8 +20,10 @@ package org.apache.atlas.repository.store.graph; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream; import java.util.List; +import java.util.Map; public interface EntityGraphDiscovery { @@ -33,7 +35,7 @@ public interface EntityGraphDiscovery { * Resolved references already exist in the ATLAS repository and have an assigned unique GUID * Unresolved attribute references result in an error if they are not composite (managed by a parent entity) */ - EntityGraphDiscoveryContext discoverEntities(List entities) throws AtlasBaseException; + EntityGraphDiscoveryContext discoverEntities() throws AtlasBaseException; void cleanUp() throws AtlasBaseException; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java index cd92a11..ca62fdc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java @@ -17,179 +17,117 @@ */ package org.apache.atlas.repository.store.graph; -import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v1.EntityStream; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; public final class EntityGraphDiscoveryContext { private static final Logger LOG = LoggerFactory.getLogger(EntityGraphDiscoveryContext.class); - private final AtlasTypeRegistry typeRegistry; - private List rootEntities = new ArrayList<>(); - private Map resolvedIds = new LinkedHashMap<>(); - private Set unresolvedIds = new HashSet<>(); - private List unresolvedIdsByUniqAttribs = new ArrayList<>(); + private final AtlasTypeRegistry typeRegistry; + private final EntityStream entityStream; + private final List referencedGuids = new ArrayList<>(); + private final Set referencedByUniqAttribs = new HashSet<>(); + private final Map resolvedGuids = new HashMap<>(); + private final Map resolvedIdsByUniqAttribs = new HashMap<>(); + private final Set localGuids = new HashSet<>(); - public EntityGraphDiscoveryContext(AtlasTypeRegistry typeRegistry) { + public EntityGraphDiscoveryContext(AtlasTypeRegistry typeRegistry, EntityStream entityStream) { this.typeRegistry = typeRegistry; + this.entityStream = entityStream; } - - public Collection getRootEntities() { - return rootEntities; + public EntityStream getEntityStream() { + return entityStream; } - public Map getResolvedIds() { - return resolvedIds; - } + public List getReferencedGuids() { return referencedGuids; } - public Set getUnresolvedIds() { - return unresolvedIds; - } + public Set getReferencedByUniqAttribs() { return referencedByUniqAttribs; } - public List getUnresolvedIdsByUniqAttribs() { - return unresolvedIdsByUniqAttribs; + public Map getResolvedGuids() { + return resolvedGuids; } - - public void addRootEntity(AtlasEntity rootEntity) { - this.rootEntities.add(rootEntity); + public Map getResolvedIdsByUniqAttribs() { + return resolvedIdsByUniqAttribs; } + public Set getLocalGuids() { return localGuids; } - public void addResolvedId(AtlasObjectId objId, AtlasVertex vertex) { - if (LOG.isDebugEnabled()) { - LOG.debug("addResolvedId({})", objId); - } - - resolvedIds.put(objId, vertex); - } - public boolean removeUnResolvedId(AtlasObjectId objId) { - if (LOG.isDebugEnabled()) { - LOG.debug("removeUnResolvedId({})", objId); + public void addReferencedGuid(String guid) { + if (! referencedGuids.contains(guid)) { + referencedGuids.add(guid); } - - return unresolvedIds.remove(objId); } + public void addReferencedByUniqAttribs(AtlasObjectId objId) { referencedByUniqAttribs.add(objId); } - public void addUnResolvedId(AtlasObjectId objId) { - if (LOG.isDebugEnabled()) { - LOG.debug("addUnResolvedId({})", objId); - } - this.unresolvedIds.add(objId); - } + public void addResolvedGuid(String guid, AtlasVertex vertex) { resolvedGuids.put(guid, vertex); } - public boolean removeUnResolvedIds(List objIds) { - if (LOG.isDebugEnabled()) { - LOG.debug("removeUnResolvedIds({})", objIds); - } + public void addResolvedIdByUniqAttribs(AtlasObjectId objId, AtlasVertex vertex) { resolvedIdsByUniqAttribs.put(objId, vertex); } - return unresolvedIds.removeAll(objIds); - } + public void addLocalGuidReference(String guid) { localGuids.add(guid); } + public boolean isResolvedGuid(String guid) { return resolvedGuids.containsKey(guid); } - public void addUnresolvedIdByUniqAttribs(AtlasObjectId objId) { - if (LOG.isDebugEnabled()) { - LOG.debug("addUnresolvedIdByUniqAttribs({})", objId); - } + public boolean isResolvedIdByUniqAttrib(AtlasObjectId objId) { return resolvedIdsByUniqAttribs.containsKey(objId); } - this.unresolvedIdsByUniqAttribs.add(objId); - } - public boolean removeUnresolvedIdsByUniqAttribs(List objIds) { - if (LOG.isDebugEnabled()) { - LOG.debug("removeUnresolvedIdsByUniqAttribs({})", objIds); - } + public AtlasVertex getResolvedEntityVertex(String guid) throws AtlasBaseException { + AtlasVertex ret = resolvedGuids.get(guid); - return unresolvedIdsByUniqAttribs.removeAll(objIds); + return ret; } - public boolean hasUnresolvedReferences() { - return unresolvedIdsByUniqAttribs.size() > 0 || unresolvedIds.size() > 0; - } - - public boolean isResolvedId(AtlasObjectId id) { - return resolvedIds.containsKey(id); - } - - public AtlasVertex getResolvedEntityVertex(AtlasObjectId ref) throws AtlasBaseException { - AtlasVertex vertex = resolvedIds.get(ref); + public AtlasVertex getResolvedEntityVertex(AtlasObjectId objId) { + AtlasVertex vertex = resolvedIdsByUniqAttribs.get(objId); // check also for sub-types; ref={typeName=Asset; guid=abcd} should match {typeName=hive_table; guid=abcd} if (vertex == null) { - final AtlasEntityType entityType = typeRegistry.getEntityTypeByName(ref.getTypeName()); + final AtlasEntityType entityType = typeRegistry.getEntityTypeByName(objId.getTypeName()); final Set allSubTypes = entityType.getAllSubTypes(); for (String subType : allSubTypes) { - AtlasObjectId subTypeObjId = new AtlasObjectId(subType, ref.getGuid(), ref.getUniqueAttributes()); + AtlasObjectId subTypeObjId = new AtlasObjectId(objId.getGuid(), subType, objId.getUniqueAttributes()); - vertex = resolvedIds.get(subTypeObjId); + vertex = resolvedIdsByUniqAttribs.get(subTypeObjId); if (vertex != null) { - resolvedIds.put(ref, vertex); + resolvedIdsByUniqAttribs.put(objId, vertex); break; } } } - if (vertex == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, - " : Could not find an entity with " + ref.toString()); - } - return vertex; } - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } else if (obj == this) { - return true; - } else if (obj.getClass() != getClass()) { - return false; - } else { - EntityGraphDiscoveryContext ctx = (EntityGraphDiscoveryContext) obj; - return Objects.equals(rootEntities, ctx.getRootEntities()) && - Objects.equals(resolvedIds, ctx.getResolvedIds()) && - Objects.equals(unresolvedIdsByUniqAttribs, ctx.getUnresolvedIdsByUniqAttribs()) && - Objects.equals(unresolvedIds, ctx.getUnresolvedIds()); - } - } - - @Override - public int hashCode() { - return Objects.hash(rootEntities, resolvedIds, unresolvedIdsByUniqAttribs, unresolvedIds); - } - public StringBuilder toString(StringBuilder sb) { if (sb == null) { sb = new StringBuilder(); } sb.append("EntityGraphDiscoveryCtx{"); - sb.append("rootEntities='").append(rootEntities).append('\''); - sb.append(", resolvedIds=").append(resolvedIds); - sb.append(", unresolvedIdsByUniqAttribs='").append(unresolvedIdsByUniqAttribs).append('\''); - sb.append(", unresolvedIds='").append(unresolvedIds).append('\''); + sb.append("referencedGuids=").append(referencedGuids); + sb.append(", referencedByUniqAttribs=").append(referencedByUniqAttribs); + sb.append(", resolvedGuids='").append(resolvedGuids); + sb.append(", resolvedIdsByUniqAttribs='").append(resolvedIdsByUniqAttribs); + sb.append(", localGuids='").append(localGuids); sb.append('}'); return sb; @@ -201,9 +139,10 @@ public final class EntityGraphDiscoveryContext { } public void cleanUp() { - rootEntities.clear(); - unresolvedIdsByUniqAttribs.clear(); - resolvedIds.clear(); - unresolvedIds.clear(); + referencedGuids.clear(); + referencedByUniqAttribs.clear(); + resolvedGuids.clear(); + resolvedIdsByUniqAttribs.clear(); + localGuids.clear(); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ce20d6f5/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java index 35ddc7d..1a0ff03 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java @@ -21,10 +21,5 @@ import org.apache.atlas.exception.AtlasBaseException; public interface EntityResolver { - - void init(EntityGraphDiscoveryContext entities) throws AtlasBaseException; - - EntityGraphDiscoveryContext resolveEntityReferences() throws AtlasBaseException; - - void cleanUp() throws AtlasBaseException; + EntityGraphDiscoveryContext resolveEntityReferences(EntityGraphDiscoveryContext entities) throws AtlasBaseException; }