Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 801EE200CF3 for ; Wed, 30 Aug 2017 02:01:44 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 7EE36167C47; Wed, 30 Aug 2017 00:01:44 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 5F7FB167C43 for ; Wed, 30 Aug 2017 02:01:42 +0200 (CEST) Received: (qmail 51577 invoked by uid 500); 30 Aug 2017 00:01:40 -0000 Mailing-List: contact commits-help@atlas.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.apache.org Delivered-To: mailing list commits@atlas.apache.org Received: (qmail 51568 invoked by uid 99); 30 Aug 2017 00:01:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Aug 2017 00:01:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 16C16E96F4; Wed, 30 Aug 2017 00:01:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sarath@apache.org To: commits@atlas.apache.org Message-Id: <7748751260654bdcbc72dcb92b400aa3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: atlas git commit: ATLAS-2040: Relationship with many-to-many cardinality gives incorrect relationship attribute value Date: Wed, 30 Aug 2017 00:01:38 +0000 (UTC) archived-at: Wed, 30 Aug 2017 00:01:44 -0000 Repository: atlas Updated Branches: refs/heads/master 1b7e41f1a -> f59284adb ATLAS-2040: Relationship with many-to-many cardinality gives incorrect relationship attribute value Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f59284ad Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f59284ad Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f59284ad Branch: refs/heads/master Commit: f59284adbb3f0231fe6cc0da3a521c29f0210dba Parents: 1b7e41f Author: Sarath Subramanian Authored: Tue Aug 29 17:01:00 2017 -0700 Committer: Sarath Subramanian Committed: Tue Aug 29 17:01:00 2017 -0700 ---------------------------------------------------------------------- .../atlas/type/AtlasRelationshipType.java | 41 +++--- .../org/apache/atlas/type/AtlasStructType.java | 2 +- .../apache/atlas/TestRelationshipUtilsV2.java | 86 ++++++++--- .../atlas/repository/graph/GraphHelper.java | 72 ++++++---- .../store/graph/v1/DeleteHandlerV1.java | 17 ++- .../store/graph/v1/EntityGraphMapper.java | 102 ++++++++++--- .../store/graph/v1/EntityGraphRetriever.java | 9 +- .../AtlasRelationshipStoreHardDeleteV1Test.java | 60 +++++++- .../AtlasRelationshipStoreSoftDeleteV1Test.java | 63 +++++++- .../graph/v1/AtlasRelationshipStoreV1Test.java | 142 ++++++++++++++++--- 10 files changed, 481 insertions(+), 113 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java index 3de02d0..aa26d18 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; @@ -118,24 +119,33 @@ public class AtlasRelationshipType extends AtlasStructType { } private void addRelationshipEdgeDirection() { - AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1(); - AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2(); - AtlasAttribute end1Attribute = end1Type.getRelationshipAttribute(endDef1.getName()); - AtlasAttribute end2Attribute = end2Type.getRelationshipAttribute(endDef2.getName()); + AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1(); + AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2(); - //default relationship edge direction is end1 (out) -> end2 (in) - AtlasRelationshipEdgeDirection end1Direction = OUT; - AtlasRelationshipEdgeDirection end2Direction = IN; + if (StringUtils.equals(endDef1.getType(), endDef2.getType()) && + StringUtils.equals(endDef1.getName(), endDef2.getName())) { - if (endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) { - end2Direction = OUT; - } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) { - end1Direction = IN; - end2Direction = OUT; - } + AtlasAttribute endAttribute = end1Type.getRelationshipAttribute(endDef1.getName()); - end1Attribute.setRelationshipEdgeDirection(end1Direction); - end2Attribute.setRelationshipEdgeDirection(end2Direction); + endAttribute.setRelationshipEdgeDirection(BOTH); + } else { + AtlasAttribute end1Attribute = end1Type.getRelationshipAttribute(endDef1.getName()); + AtlasAttribute end2Attribute = end2Type.getRelationshipAttribute(endDef2.getName()); + + //default relationship edge direction is end1 (out) -> end2 (in) + AtlasRelationshipEdgeDirection end1Direction = OUT; + AtlasRelationshipEdgeDirection end2Direction = IN; + + if (endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) { + end2Direction = OUT; + } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) { + end1Direction = IN; + end2Direction = OUT; + } + + end1Attribute.setRelationshipEdgeDirection(end1Direction); + end2Attribute.setRelationshipEdgeDirection(end2Direction); + } } @Override @@ -200,7 +210,6 @@ public class AtlasRelationshipType extends AtlasStructType { AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2(); RelationshipCategory relationshipCategory = relationshipDef.getRelationshipCategory(); String name = relationshipDef.getName(); - boolean isContainer1 = endDef1.getIsContainer(); boolean isContainer2 = endDef2.getIsContainer(); http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 6f30ff3..1c202e7 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -794,6 +794,6 @@ public class AtlasStructType extends AtlasType { private static final char DOUBLE_QUOTE_CHAR = '"'; private static final char SPACE_CHAR = ' '; - public enum AtlasRelationshipEdgeDirection { IN, OUT } + public enum AtlasRelationshipEdgeDirection { IN, OUT, BOTH } } } http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java index 98be2b8..d0effd6 100755 --- a/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java @@ -23,8 +23,8 @@ import com.google.common.collect.ImmutableSet; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; +import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasClassificationDef; import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEnumDef; @@ -33,18 +33,17 @@ import org.apache.atlas.model.typedef.AtlasRelationshipDef; import org.apache.atlas.model.typedef.AtlasRelationshipEndDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.List; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.getArrayTypeName; -import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.getMapTypeName; +import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.BOTH; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.AGGREGATION; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.ASSOCIATION; @@ -72,12 +71,13 @@ public final class TestRelationshipUtilsV2 { public static final String EMPLOYEE_TYPE = "Employee"; public static final String EMPLOYEE_DEPARTMENT_TYPE = "EmployeeDepartment"; public static final String EMPLOYEE_MANAGER_TYPE = "EmployeeManager"; - public static final String EMPLOYEE_MENTOR_TYPE = "EmployeeMentor"; + public static final String EMPLOYEE_MENTORS_TYPE = "EmployeeMentors"; + public static final String EMPLOYEE_FRIENDS_TYPE = "EmployeeFriends"; + public static final String PERSON_SIBLING_TYPE = "PersonSibling"; public static final String TYPE_A = "A"; public static final String TYPE_B = "B"; public static final String DEFAULT_VERSION = "1.0"; - private TestRelationshipUtilsV2() { } public static AtlasTypesDef getDepartmentEmployeeTypes() throws AtlasBaseException { @@ -121,23 +121,36 @@ public final class TestRelationshipUtilsV2 { DEFAULT_VERSION, AGGREGATION, ONE_TO_TWO, new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "department", SINGLE), new AtlasRelationshipEndDef(DEPARTMENT_TYPE, "employees", SET, true)); + /******* [Manager -> Employee] Relationship *******/ AtlasRelationshipDef employeeManagerType = new AtlasRelationshipDef(EMPLOYEE_MANAGER_TYPE, description(EMPLOYEE_MANAGER_TYPE), DEFAULT_VERSION, AGGREGATION, ONE_TO_TWO, new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "manager", SINGLE), new AtlasRelationshipEndDef(MANAGER_TYPE, "subordinates", SET, true)); - /******* [Mentor -> Employee] Relationship *******/ - AtlasRelationshipDef employeeMentorType = new AtlasRelationshipDef(EMPLOYEE_MENTOR_TYPE, description(EMPLOYEE_MENTOR_TYPE), + /******* [Mentors -> Employee] Relationship *******/ + AtlasRelationshipDef employeeMentorsType = new AtlasRelationshipDef(EMPLOYEE_MENTORS_TYPE, description(EMPLOYEE_MENTORS_TYPE), + DEFAULT_VERSION, AGGREGATION, ONE_TO_TWO, + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentors", SET), + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentees", SET, true)); + + /******* [Friends -> Employee] Relationship *******/ + AtlasRelationshipDef employeeFriendsType = new AtlasRelationshipDef(EMPLOYEE_FRIENDS_TYPE, description(EMPLOYEE_FRIENDS_TYPE), DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO, - new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentor", SINGLE), - new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentees", SET)); + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "friends", SET), + new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "friends", SET)); + + /******* [Person -> Sibling] Relationship *******/ + AtlasRelationshipDef personSiblingType = new AtlasRelationshipDef(PERSON_SIBLING_TYPE, description(PERSON_SIBLING_TYPE), + DEFAULT_VERSION, ASSOCIATION, BOTH, + new AtlasRelationshipEndDef(PERSON_TYPE, "sibling", SINGLE), + new AtlasRelationshipEndDef(PERSON_TYPE, "sibling", SINGLE)); return new AtlasTypesDef(ImmutableList.of(orgLevelType), ImmutableList.of(addressType), ImmutableList.of(securityClearanceType), ImmutableList.of(personType, employeeType, departmentType, managerType), - ImmutableList.of(employeeDepartmentType, employeeManagerType, employeeMentorType)); + ImmutableList.of(employeeDepartmentType, employeeManagerType, employeeMentorsType, employeeFriendsType, personSiblingType)); } public static AtlasEntitiesWithExtInfo getDepartmentEmployeeInstances() { @@ -163,25 +176,30 @@ public final class TestRelationshipUtilsV2 { johnAddr.setAttribute("street", "Stewart Drive"); johnAddr.setAttribute("city", "Sunnyvale"); - /******* Manager - Jane (John and Max subordinates) *******/ + AtlasStruct mikeAddr = new AtlasStruct(ADDRESS_TYPE); + mikeAddr.setAttribute("street", "Casa Verde St"); + mikeAddr.setAttribute("city", "San Jose"); + + /******* Manager - Jane (Subordinates: [John, Max]) *******/ AtlasEntity jane = new AtlasEntity(MANAGER_TYPE); jane.setAttribute("name", "Jane"); jane.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); jane.setAttribute("address", janeAddr); - /******* Manager - Julius (no subordinates) *******/ + /******* Manager - Julius (Subordinates: [], Sibling: Jane) *******/ AtlasEntity julius = new AtlasEntity(MANAGER_TYPE); julius.setAttribute("name", "Julius"); julius.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); + julius.setRelationshipAttribute("sibling", getAtlasObjectId(jane)); julius.setAttribute("address", juliusAddr); - /******* Employee - Max (Manager: Jane, Mentor: Julius) *******/ + /******* Employee - Max (Manager: Jane, Mentors: [Julius], Sibling: Julius) *******/ AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE); max.setAttribute("name", "Max"); max.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); max.setAttribute("address", maxAddr); max.setRelationshipAttribute("manager", getAtlasObjectId(jane)); - max.setRelationshipAttribute("mentor", getAtlasObjectId(julius)); + max.setRelationshipAttribute("mentors", getAtlasObjectIds(julius)); max.setAttribute("birthday",new Date(1979, 3, 15)); max.setAttribute("hasPets", true); max.setAttribute("age", 36); @@ -193,13 +211,14 @@ public final class TestRelationshipUtilsV2 { max.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000")); max.setAttribute("approximationOfPi", new BigDecimal("3.1415926535897932")); - /******* Employee - John (Manager: Jane, Mentor: Max) *******/ + /******* Employee - John (Manager: Jane, Mentors: [Max], Friends: [Max]) *******/ AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE); john.setAttribute("name", "John"); john.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); john.setAttribute("address", johnAddr); john.setRelationshipAttribute("manager", getAtlasObjectId(jane)); - john.setRelationshipAttribute("mentor", getAtlasObjectId(max)); + john.setRelationshipAttribute("mentors", getAtlasObjectIds(max, julius)); + john.setRelationshipAttribute("friends", getAtlasObjectIds(max)); john.setAttribute("birthday",new Date(1950, 5, 15)); john.setAttribute("hasPets", true); john.setAttribute("numberOfCars", 1); @@ -211,11 +230,30 @@ public final class TestRelationshipUtilsV2 { john.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000")); john.setAttribute("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); + /******* Employee - Mike (Manager: Julius, Friends: [Max, John]) *******/ + AtlasEntity mike = new AtlasEntity(EMPLOYEE_TYPE); + mike.setAttribute("name", "Mike"); + mike.setRelationshipAttribute("department", getAtlasObjectId(hrDept)); + mike.setAttribute("address", mikeAddr); + mike.setRelationshipAttribute("manager", getAtlasObjectId(julius)); + mike.setRelationshipAttribute("friends", getAtlasObjectIds(max, john)); + mike.setAttribute("birthday",new Date(1947, 8, 15)); + mike.setAttribute("hasPets", false); + mike.setAttribute("numberOfCars", 2); + mike.setAttribute("houseNumber", 3737); + mike.setAttribute("carMileage", 25000); + mike.setAttribute("shares", Long.MIN_VALUE); + mike.setAttribute("salary", Double.MIN_VALUE); + mike.setAttribute("age", 37); + mike.setAttribute("numberOfStarsEstimate", new BigInteger("5000050000050000050005")); + mike.setAttribute("approximationOfPi", new BigDecimal("3.14159")); + ret.addEntity(hrDept); ret.addEntity(jane); ret.addEntity(julius); ret.addEntity(max); ret.addEntity(john); + ret.addEntity(mike); return ret; } @@ -264,4 +302,16 @@ public final class TestRelationshipUtilsV2 { private static ImmutableSet superType(String superTypeName) { return StringUtils.isNotEmpty(superTypeName) ? ImmutableSet.of(superTypeName) : ImmutableSet.of(); } + + private static List getAtlasObjectIds(AtlasEntity... entities) { + List ret = new ArrayList<>(); + + if (ArrayUtils.isNotEmpty(entities)) { + for (AtlasEntity entity : entities) { + ret.add(getAtlasObjectId(entity)); + } + } + + return ret; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 0177f7e..020dd45 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -42,7 +42,6 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasRelationshipType; -import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasType; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.ITypedInstance; @@ -86,6 +85,10 @@ import java.util.Set; import java.util.Stack; import java.util.UUID; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; + /** * Utility class for graph operations. */ @@ -308,7 +311,7 @@ public final class GraphHelper { //In some cases of parallel APIs, the edge is added, but get edge by label doesn't return the edge. ATLAS-1104 //So traversing all the edges - public Iterator getAdjacentEdgesByLabel(AtlasVertex instanceVertex, AtlasEdgeDirection direction, final String edgeLabel) { + public static Iterator getAdjacentEdgesByLabel(AtlasVertex instanceVertex, AtlasEdgeDirection direction, final String edgeLabel) { if (LOG.isDebugEnabled()) { LOG.debug("Finding edges for {} with label {}", string(instanceVertex), edgeLabel); } @@ -348,11 +351,11 @@ public final class GraphHelper { return null; } - public Iterator getIncomingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) { + public static Iterator getIncomingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) { return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.IN, edgeLabel); } - public Iterator getOutGoingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) { + public static Iterator getOutGoingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) { return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel); } @@ -361,20 +364,24 @@ public final class GraphHelper { switch (edgeDirection) { case IN: - ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.IN); - break; + ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.IN); + break; case OUT: + ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.OUT); + break; + + case BOTH: default: - ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.OUT); + ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.BOTH); break; } return ret; } - public Iterator getEdgesForLabel(AtlasVertex vertex, String edgeLabel, AtlasRelationshipEdgeDirection edgeDirection) { - Iterator ret; + public static Iterator getEdgesForLabel(AtlasVertex vertex, String edgeLabel, AtlasRelationshipEdgeDirection edgeDirection) { + Iterator ret = null; switch (edgeDirection) { case IN: @@ -382,8 +389,11 @@ public final class GraphHelper { break; case OUT: - default: - ret = getOutGoingEdgesByLabel(vertex, edgeLabel); + ret = getOutGoingEdgesByLabel(vertex, edgeLabel); + break; + + case BOTH: + ret = getAdjacentEdgesByLabel(vertex, AtlasEdgeDirection.BOTH, edgeLabel); break; } @@ -1341,32 +1351,38 @@ public final class GraphHelper { return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false; } - public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection) { + public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection, + AtlasVertex parentVertex) { AtlasObjectId ret = null; - if (relationshipDirection == AtlasRelationshipEdgeDirection.OUT) { - ret = new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex())); + if (relationshipDirection == OUT) { + ret = getAtlasObjectIdForInVertex(edge); + + } else if (relationshipDirection == IN) { + ret = getAtlasObjectIdForOutVertex(edge); - } else if (relationshipDirection == AtlasRelationshipEdgeDirection.IN) { - ret = new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex())); + } else if (relationshipDirection == BOTH){ + // since relationship direction is BOTH, edge direction can be inward or outward + // compare with parent entity vertex and pick the right reference vertex + if (verticesEquals(parentVertex, edge.getOutVertex())) { + ret = getAtlasObjectIdForInVertex(edge); + } else { + ret = getAtlasObjectIdForOutVertex(edge); + } } return ret; } - public static AtlasObjectId getCurrentObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection) { - String typeName = null; - String guid = null; - - if (relationshipDirection == AtlasRelationshipEdgeDirection.OUT) { - typeName = GraphHelper.getTypeName(edge.getOutVertex()); - guid = GraphHelper.getGuid(edge.getOutVertex()); + public static AtlasObjectId getAtlasObjectIdForOutVertex(AtlasEdge edge) { + return new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex())); + } - } else if (relationshipDirection == AtlasRelationshipEdgeDirection.IN) { - typeName = GraphHelper.getTypeName(edge.getInVertex()); - guid = GraphHelper.getGuid(edge.getInVertex()); - } + public static AtlasObjectId getAtlasObjectIdForInVertex(AtlasEdge edge) { + return new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex())); + } - return new AtlasObjectId(guid, typeName); + private static boolean verticesEquals(AtlasVertex vertexA, AtlasVertex vertexB) { + return StringUtils.equals(getGuid(vertexB), getGuid(vertexA)); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index 4271376..b0940f6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -40,6 +40,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,9 +55,13 @@ import java.util.Set; import java.util.Stack; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; +import static org.apache.atlas.repository.graph.GraphHelper.getAtlasObjectIdForInVertex; +import static org.apache.atlas.repository.graph.GraphHelper.getAtlasObjectIdForOutVertex; +import static org.apache.atlas.repository.graph.GraphHelper.getGuid; import static org.apache.atlas.repository.graph.GraphHelper.getReferenceObjectId; import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.string; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; public abstract class DeleteHandlerV1 { @@ -219,14 +224,14 @@ public abstract class DeleteHandlerV1 { * @throws AtlasException */ public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, - boolean forceDeleteStructTrait) throws AtlasBaseException { + boolean forceDeleteStructTrait, AtlasVertex vertex) throws AtlasBaseException { // default edge direction is outward - return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT); + return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT, vertex); } public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait, - AtlasRelationshipEdgeDirection relationshipDirection) throws AtlasBaseException { + AtlasRelationshipEdgeDirection relationshipDirection, AtlasVertex entityVertex) throws AtlasBaseException { LOG.debug("Deleting {}", string(edge)); boolean forceDelete = (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait; @@ -250,7 +255,7 @@ public abstract class DeleteHandlerV1 { if (isRelationshipEdge(edge)) { deleteEdge(edge, false); - AtlasObjectId deletedReferenceObjectId = getReferenceObjectId(edge, relationshipDirection); + AtlasObjectId deletedReferenceObjectId = getReferenceObjectId(edge, relationshipDirection, entityVertex); RequestContextV1.get().recordEntityUpdate(deletedReferenceObjectId); } else { //legacy case - not a relationship edge @@ -344,7 +349,7 @@ public abstract class DeleteHandlerV1 { if (edges != null) { while (edges.hasNext()) { AtlasEdge edge = edges.next(); - deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false); + deleteEdgeReference(edge, elemType.getTypeCategory(), isOwned, false, instanceVertex); } } } @@ -377,7 +382,7 @@ public abstract class DeleteHandlerV1 { boolean isOwned) throws AtlasBaseException { AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel); if (edge != null) { - deleteEdgeReference(edge, typeCategory, isOwned, false); + deleteEdgeReference(edge, typeCategory, isOwned, false, outVertex); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index fbb9858..9700917 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -51,6 +51,7 @@ import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.IteratorUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -70,7 +71,9 @@ import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.string; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; @Component public class EntityGraphMapper { @@ -353,7 +356,7 @@ public class EntityGraphMapper { AtlasEdge newEdge = mapStructValue(ctx, context); if (currentEdge != null && !currentEdge.equals(newEdge)) { - deleteHandler.deleteEdgeReference(currentEdge, ctx.getAttrType().getTypeCategory(), false, true); + deleteHandler.deleteEdgeReference(currentEdge, ctx.getAttrType().getTypeCategory(), false, true, ctx.getReferringVertex()); } return newEdge; @@ -423,7 +426,7 @@ public class EntityGraphMapper { //delete old reference deleteHandler.deleteEdgeReference(currentEdge, ctx.getAttrType().getTypeCategory(), ctx.getAttribute().isOwnedRef(), - true, ctx.getAttribute().getRelationshipEdgeDirection()); + true, ctx.getAttribute().getRelationshipEdgeDirection(), ctx.getReferringVertex()); } return newEdge; @@ -457,7 +460,7 @@ public class EntityGraphMapper { if (!inverseEdge.equals(newEdge)) { // Disconnect old reference deleteHandler.deleteEdgeReference(inverseEdge, inverseAttribute.getAttributeType().getTypeCategory(), - inverseAttribute.isOwnedRef(), true); + inverseAttribute.isOwnedRef(), true, inverseVertex); } else { // Edge already exists for this attribute between these vertices. @@ -675,9 +678,7 @@ public class EntityGraphMapper { Map relationshipAttributes = getRelationshipAttributes(ctx.getValue()); if (ctx.getCurrentEdge() != null) { - ret = updateRelationship(ctx.getCurrentEdge(), attributeVertex, edgeDirection, relationshipAttributes); - - recordEntityUpdate(attributeVertex); + ret = updateRelationship(ctx.getCurrentEdge(), entityVertex, attributeVertex, edgeDirection, relationshipAttributes); } else { String relationshipName = graphHelper.getRelationshipDefName(entityVertex, entityType, attributeName); @@ -805,10 +806,16 @@ public class EntityGraphMapper { List newElements = (List) ctx.getValue(); AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); AtlasType elementType = arrType.getElementType(); - List currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty()); boolean isReference = AtlasGraphUtilsV1.isReference(elementType); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); List newElementsCreated = new ArrayList<>(); + List currentElements; + + if (isRelationshipAttribute(attribute)) { + currentElements = getArrayElementsUsingRelationship(ctx.getReferringVertex(), attribute, elementType); + } else { + currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty()); + } if (CollectionUtils.isNotEmpty(newElements)) { for (int index = 0; index < newElements.size(); index++) { @@ -817,6 +824,7 @@ public class EntityGraphMapper { ctx.getVertexProperty(), elementType, existingEdge); Object newEntry = mapCollectionElementsToVertex(arrCtx, context); + if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) { // Update the inverse reference value. AtlasEdge newEdge = (AtlasEdge) newEntry; @@ -829,7 +837,7 @@ public class EntityGraphMapper { } if (isReference) { - List additionalEdges = removeUnusedArrayEntries(attribute, (List) currentElements, (List) newElementsCreated); + List additionalEdges = removeUnusedArrayEntries(attribute, (List) currentElements, (List) newElementsCreated, ctx.getReferringVertex()); newElementsCreated.addAll(additionalEdges); } @@ -1048,7 +1056,7 @@ public class EntityGraphMapper { AtlasEdge currentEdge = (AtlasEdge)currentMap.get(currentKey); if (!newMap.values().contains(currentEdge)) { - boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true); + boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex); if (!deleted) { additionalMap.put(currentKey, currentEdge); @@ -1104,8 +1112,10 @@ public class EntityGraphMapper { return newEdge; } - private AtlasEdge updateRelationship(AtlasEdge currentEdge, final AtlasVertex newEntityVertex, AtlasRelationshipEdgeDirection edgeDirection, - Map relationshipAttributes) throws AtlasBaseException { + + private AtlasEdge updateRelationship(AtlasEdge currentEdge, final AtlasVertex parentEntityVertex, final AtlasVertex newEntityVertex, + AtlasRelationshipEdgeDirection edgeDirection, Map relationshipAttributes) + throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("Updating entity reference using relationship {} for reference attribute {}", getTypeName(newEntityVertex)); } @@ -1115,8 +1125,15 @@ public class EntityGraphMapper { // Max's mentor updated from John to Jane (John.mentee --> Max.mentor) // mentor attribute (IN direction), current mentee vertex (John) (OUT vertex) - String currentEntityId = (edgeDirection == IN) ? getIdFromVertex(currentEdge.getOutVertex()) : - getIdFromVertex(currentEdge.getInVertex()); + String currentEntityId; + + if (edgeDirection == IN) { + currentEntityId = getIdFromOutVertex(currentEdge); + } else if (edgeDirection == OUT) { + currentEntityId = getIdFromInVertex(currentEdge); + } else { + currentEntityId = getIdFromBothVertex(currentEdge, parentEntityVertex); + } String newEntityId = getIdFromVertex(newEntityVertex); AtlasEdge ret = currentEdge; @@ -1129,8 +1146,17 @@ public class EntityGraphMapper { relationshipName = currentEdge.getLabel(); } - ret = (edgeDirection == IN) ? getOrCreateRelationship(newEntityVertex, currentEdge.getInVertex(), relationshipName, relationshipAttributes) : - getOrCreateRelationship(currentEdge.getOutVertex(), newEntityVertex, relationshipName, relationshipAttributes); + if (edgeDirection == IN) { + ret = getOrCreateRelationship(newEntityVertex, currentEdge.getInVertex(), relationshipName, relationshipAttributes); + + } else if (edgeDirection == OUT) { + ret = getOrCreateRelationship(currentEdge.getOutVertex(), newEntityVertex, relationshipName, relationshipAttributes); + } else { + ret = getOrCreateRelationship(newEntityVertex, parentEntityVertex, relationshipName, relationshipAttributes); + } + + //record entity update on new relationship vertex + recordEntityUpdate(newEntityVertex); } return ret; @@ -1145,6 +1171,21 @@ public class EntityGraphMapper { } } + public static List getArrayElementsUsingRelationship(AtlasVertex vertex, AtlasAttribute attribute, AtlasType elementType) { + List ret = null; + + if (AtlasGraphUtilsV1.isReference(elementType)) { + + AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection(); + String edgeLabel = attribute.getRelationshipEdgeLabel(); + + Iterator edgesForLabel = GraphHelper.getEdgesForLabel(vertex, edgeLabel, edgeDirection); + + ret = IteratorUtils.toList(edgesForLabel); + } + return ret; + } + private AtlasEdge getEdgeAt(List currentElements, int index, AtlasType elemType) { AtlasEdge ret = null; @@ -1158,7 +1199,8 @@ public class EntityGraphMapper { } //Removes unused edges from the old collection, compared to the new collection - private List removeUnusedArrayEntries(AtlasAttribute attribute, List currentEntries, List newEntries) throws AtlasBaseException { + + private List removeUnusedArrayEntries(AtlasAttribute attribute, List currentEntries, List newEntries, AtlasVertex entityVertex) throws AtlasBaseException { if (CollectionUtils.isNotEmpty(currentEntries)) { AtlasType entryType = ((AtlasArrayType) attribute.getAttributeType()).getElementType(); @@ -1170,7 +1212,7 @@ public class EntityGraphMapper { for (AtlasEdge edge : edgesToRemove) { boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(), attribute.isOwnedRef(), - true, attribute.getRelationshipEdgeDirection()); + true, attribute.getRelationshipEdgeDirection(), entityVertex); if (!deleted) { additionalElements.add(edge); @@ -1184,7 +1226,6 @@ public class EntityGraphMapper { return Collections.emptyList(); } - private void setArrayElementsProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, List values) { if (AtlasGraphUtilsV1.isReference(elementType)) { GraphHelper.setListPropertyFromElementIds(vertex, vertexPropertyName, (List) values); @@ -1334,7 +1375,7 @@ public class EntityGraphMapper { String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, classificationName); AtlasEdge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel); if (edge != null) { - deleteHandler.deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, true); + deleteHandler.deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, true, instanceVertex); // update the traits in entity once trait removal is successful traitNames.remove(classificationName); @@ -1433,7 +1474,7 @@ public class EntityGraphMapper { private static void compactAttributes(AtlasEntity entity) { if (entity != null) { Map relationshipAttributes = entity.getRelationshipAttributes(); - Map attributes = entity.getAttributes(); + Map attributes = entity.getAttributes(); if (MapUtils.isNotEmpty(relationshipAttributes) && MapUtils.isNotEmpty(attributes)) { for (String attrName : relationshipAttributes.keySet()) { @@ -1444,4 +1485,23 @@ public class EntityGraphMapper { } } } -} + + private String getIdFromInVertex(AtlasEdge edge) { + return getIdFromVertex(edge.getInVertex()); + } + + private String getIdFromOutVertex(AtlasEdge edge) { + return getIdFromVertex(edge.getOutVertex()); + } + + private String getIdFromBothVertex(AtlasEdge currentEdge, AtlasVertex parentEntityVertex) { + String parentEntityId = getIdFromVertex(parentEntityVertex); + String currentEntityId = getIdFromVertex(currentEdge.getInVertex()); + + if (StringUtils.equals(currentEntityId, parentEntityId)) { + currentEntityId = getIdFromOutVertex(currentEdge); + } + + return currentEntityId; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index 4e7aa47..f6aa7bb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -72,6 +72,9 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; +import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; public final class EntityGraphRetriever { @@ -683,10 +686,12 @@ public final class EntityGraphRetriever { List ret = new ArrayList<>(); Iterator edges = null; - if (attribute.getRelationshipEdgeDirection() == AtlasRelationshipEdgeDirection.IN) { + if (attribute.getRelationshipEdgeDirection() == IN) { edges = graphHelper.getIncomingEdgesByLabel(entityVertex, attribute.getRelationshipEdgeLabel()); - } else if (attribute.getRelationshipEdgeDirection() == AtlasRelationshipEdgeDirection.OUT) { + } else if (attribute.getRelationshipEdgeDirection() == OUT) { edges = graphHelper.getOutGoingEdgesByLabel(entityVertex, attribute.getRelationshipEdgeLabel()); + } else if (attribute.getRelationshipEdgeDirection() == BOTH) { + edges = graphHelper.getAdjacentEdgesByLabel(entityVertex, AtlasEdgeDirection.BOTH, attribute.getRelationshipEdgeLabel()); } if (edges != null) { http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java index 2c31140..109118e 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java @@ -20,9 +20,15 @@ package org.apache.atlas.repository.store.graph.v1; import com.google.common.collect.ImmutableList; import org.apache.atlas.TestModules; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; import org.testng.annotations.Guice; +import java.util.List; + import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; /** * Inverse reference update test with {@link HardDeleteHandlerV1} @@ -51,4 +57,56 @@ public class AtlasRelationshipStoreHardDeleteV1Test extends AtlasRelationshipSto protected void verifyRelationshipAttributeUpdate_NonComposite_OneToOne(AtlasEntity a1, AtlasEntity b) { verifyRelationshipAttributeValue(a1, "b", null); } -} + + @Override + protected void verifyRelationshipAttributeUpdate_ManyToMany_Friends(AtlasEntity max, AtlasEntity julius, AtlasEntity mike, AtlasEntity john) throws Exception { + AtlasObjectId johnId = employeeNameIdMap.get("John"); + AtlasObjectId mikeId = employeeNameIdMap.get("Mike"); + AtlasObjectId juliusId = employeeNameIdMap.get("Julius"); + AtlasObjectId maxId = employeeNameIdMap.get("Max"); + + List maxFriendsIds = toAtlasObjectIds(max.getRelationshipAttribute("friends")); + assertNotNull(maxFriendsIds); + assertEquals(maxFriendsIds.size(), 2); + assertObjectIdsContains(maxFriendsIds, johnId); + assertObjectIdsContains(maxFriendsIds, juliusId); + + // Julius's updated friends: [Max] + List juliusFriendsIds = toAtlasObjectIds(julius.getRelationshipAttribute("friends")); + assertNotNull(juliusFriendsIds); + assertEquals(juliusFriendsIds.size(), 1); + assertObjectIdsContains(juliusFriendsIds, maxId); + + // Mike's updated friends: [John] + List mikeFriendsIds = toAtlasObjectIds(mike.getRelationshipAttribute("friends")); + assertNotNull(mikeFriendsIds); + assertEquals(mikeFriendsIds.size(), 1); + assertObjectIdsContains(mikeFriendsIds, johnId); + + // John's updated friends: [Max, Mike] + List johnFriendsIds = toAtlasObjectIds(john.getRelationshipAttribute("friends")); + assertNotNull(johnFriendsIds); + assertEquals(johnFriendsIds.size(), 2); + assertObjectIdsContains(johnFriendsIds, maxId); + assertObjectIdsContains(johnFriendsIds, mikeId); + } + + protected void verifyRelationshipAttributeUpdate_OneToOne_Sibling(AtlasEntity julius, AtlasEntity jane, AtlasEntity mike) throws Exception { + AtlasObjectId juliusId = employeeNameIdMap.get("Julius"); + AtlasObjectId mikeId = employeeNameIdMap.get("Mike"); + + // Julius sibling updated to Mike + AtlasObjectId juliusSiblingId = toAtlasObjectId(julius.getRelationshipAttribute("sibling")); + assertNotNull(juliusSiblingId); + assertObjectIdEquals(juliusSiblingId, mikeId); + + // Mike's sibling is Julius + AtlasObjectId mikeSiblingId = toAtlasObjectId(mike.getRelationshipAttribute("sibling")); + assertNotNull(mikeSiblingId); + assertObjectIdEquals(mikeSiblingId, juliusId); + + // Julius removed from Jane's sibling (hard delete) + AtlasObjectId janeSiblingId = toAtlasObjectId(jane.getRelationshipAttribute("sibling")); + assertNull(janeSiblingId); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java index 33ef8c0..4faf5ad 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java @@ -20,9 +20,14 @@ package org.apache.atlas.repository.store.graph.v1; import com.google.common.collect.ImmutableList; import org.apache.atlas.TestModules; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; import org.testng.annotations.Guice; +import java.util.List; + import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; /** @@ -52,4 +57,60 @@ public class AtlasRelationshipStoreSoftDeleteV1Test extends AtlasRelationshipSto protected void verifyRelationshipAttributeUpdate_NonComposite_OneToOne(AtlasEntity a1, AtlasEntity b) { verifyRelationshipAttributeValue(a1, "b", b.getGuid()); } -} + + @Override + protected void verifyRelationshipAttributeUpdate_ManyToMany_Friends(AtlasEntity max, AtlasEntity julius, AtlasEntity mike, AtlasEntity john) throws Exception { + AtlasObjectId johnId = employeeNameIdMap.get("John"); + AtlasObjectId mikeId = employeeNameIdMap.get("Mike"); + AtlasObjectId juliusId = employeeNameIdMap.get("Julius"); + AtlasObjectId maxId = employeeNameIdMap.get("Max"); + + // Max's updated friends: [Julius, John, Mike(soft deleted)] + List maxFriendsIds = toAtlasObjectIds(max.getRelationshipAttribute("friends")); + assertNotNull(maxFriendsIds); + assertEquals(maxFriendsIds.size(), 3); + assertObjectIdsContains(maxFriendsIds, johnId); + assertObjectIdsContains(maxFriendsIds, juliusId); + assertObjectIdsContains(maxFriendsIds, mikeId); + + // Julius's updated friends: [Max] + List juliusFriendsIds = toAtlasObjectIds(julius.getRelationshipAttribute("friends")); + assertNotNull(juliusFriendsIds); + assertEquals(juliusFriendsIds.size(), 1); + assertObjectIdsContains(juliusFriendsIds, maxId); + + // Mike's updated friends: [John, Max(soft deleted)] + List mikeFriendsIds = toAtlasObjectIds(mike.getRelationshipAttribute("friends")); + assertNotNull(mikeFriendsIds); + assertEquals(mikeFriendsIds.size(), 2); + assertObjectIdsContains(mikeFriendsIds, johnId); + assertObjectIdsContains(mikeFriendsIds, maxId); + + // John's updated friends: [Max, Mike] + List johnFriendsIds = toAtlasObjectIds(john.getRelationshipAttribute("friends")); + assertNotNull(johnFriendsIds); + assertEquals(johnFriendsIds.size(), 2); + assertObjectIdsContains(johnFriendsIds, maxId); + assertObjectIdsContains(johnFriendsIds, mikeId); + } + + protected void verifyRelationshipAttributeUpdate_OneToOne_Sibling(AtlasEntity julius, AtlasEntity jane, AtlasEntity mike) throws Exception { + AtlasObjectId juliusId = employeeNameIdMap.get("Julius"); + AtlasObjectId mikeId = employeeNameIdMap.get("Mike"); + + // Julius sibling updated to Mike + AtlasObjectId juliusSiblingId = toAtlasObjectId(julius.getRelationshipAttribute("sibling")); + assertNotNull(juliusSiblingId); + assertObjectIdEquals(juliusSiblingId, mikeId); + + // Mike's sibling is Julius + AtlasObjectId mikeSiblingId = toAtlasObjectId(mike.getRelationshipAttribute("sibling")); + assertNotNull(mikeSiblingId); + assertObjectIdEquals(mikeSiblingId, juliusId); + + // Jane's sibling is still Julius (soft delete) + AtlasObjectId janeSiblingId = toAtlasObjectId(jane.getRelationshipAttribute("sibling")); + assertNotNull(janeSiblingId); + assertObjectIdEquals(janeSiblingId, juliusId); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/f59284ad/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java index a35647d..94cc5b9 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java @@ -131,21 +131,24 @@ public abstract class AtlasRelationshipStoreV1Test { AtlasObjectId johnId = employeeNameIdMap.get("John"); AtlasObjectId juliusId = employeeNameIdMap.get("Julius"); AtlasObjectId janeId = employeeNameIdMap.get("Jane"); + AtlasObjectId mikeId = employeeNameIdMap.get("Mike"); AtlasEntity hrDept = getEntityFromStore(hrId.getGuid()); AtlasEntity max = getEntityFromStore(maxId.getGuid()); AtlasEntity john = getEntityFromStore(johnId.getGuid()); AtlasEntity julius = getEntityFromStore(juliusId.getGuid()); AtlasEntity jane = getEntityFromStore(janeId.getGuid()); + AtlasEntity mike = getEntityFromStore(mikeId.getGuid()); // Department relationship attributes List deptEmployees = toAtlasObjectIds(hrDept.getRelationshipAttribute("employees")); assertNotNull(deptEmployees); - assertEquals(deptEmployees.size(), 4); + assertEquals(deptEmployees.size(), 5); assertObjectIdsContains(deptEmployees, maxId); assertObjectIdsContains(deptEmployees, johnId); assertObjectIdsContains(deptEmployees, juliusId); assertObjectIdsContains(deptEmployees, janeId); + assertObjectIdsContains(deptEmployees, mikeId); // Max employee validation AtlasObjectId maxDepartmentId = toAtlasObjectId(max.getRelationshipAttribute("department")); @@ -156,15 +159,22 @@ public abstract class AtlasRelationshipStoreV1Test { assertNotNull(maxManagerId); assertObjectIdEquals(maxManagerId, janeId); - AtlasObjectId maxMentorId = toAtlasObjectId(max.getRelationshipAttribute("mentor")); - assertNotNull(maxMentorId); - assertObjectIdEquals(maxMentorId, juliusId); + List maxMentorsId = toAtlasObjectIds(max.getRelationshipAttribute("mentors")); + assertNotNull(maxMentorsId); + assertEquals(maxMentorsId.size(), 1); + assertObjectIdEquals(maxMentorsId.get(0), juliusId); List maxMenteesId = toAtlasObjectIds(max.getRelationshipAttribute("mentees")); assertNotNull(maxMenteesId); assertEquals(maxMenteesId.size(), 1); assertObjectIdEquals(maxMenteesId.get(0), johnId); + List maxFriendsIds = toAtlasObjectIds(max.getRelationshipAttribute("friends")); + assertNotNull(maxFriendsIds); + assertEquals(maxFriendsIds.size(), 2); + assertObjectIdsContains(maxFriendsIds, mikeId); + assertObjectIdsContains(maxFriendsIds, johnId); + // John Employee validation AtlasObjectId johnDepartmentId = toAtlasObjectId(john.getRelationshipAttribute("department")); assertNotNull(johnDepartmentId); @@ -174,13 +184,42 @@ public abstract class AtlasRelationshipStoreV1Test { assertNotNull(johnManagerId); assertObjectIdEquals(johnManagerId, janeId); - AtlasObjectId johnMentorId = toAtlasObjectId(john.getRelationshipAttribute("mentor")); - assertNotNull(johnMentorId); - assertObjectIdEquals(johnMentorId, maxId); + List johnMentorIds = toAtlasObjectIds(john.getRelationshipAttribute("mentors")); + assertNotNull(johnMentorIds); + assertEquals(johnMentorIds.size(), 2); + assertObjectIdsContains(johnMentorIds, maxId); + assertObjectIdsContains(johnMentorIds, juliusId); List johnMenteesId = toAtlasObjectIds(john.getRelationshipAttribute("mentees")); assertEmpty(johnMenteesId); + List johnFriendsIds = toAtlasObjectIds(john.getRelationshipAttribute("friends")); + assertNotNull(johnFriendsIds); + assertEquals(johnFriendsIds.size(), 2); + assertObjectIdsContains(johnFriendsIds, mikeId); + assertObjectIdsContains(johnFriendsIds, maxId); + + // Mike Employee validation + AtlasObjectId mikeDepartmentId = toAtlasObjectId(mike.getRelationshipAttribute("department")); + assertNotNull(mikeDepartmentId); + assertObjectIdEquals(mikeDepartmentId, hrId); + + AtlasObjectId mikeManagerId = toAtlasObjectId(mike.getRelationshipAttribute("manager")); + assertNotNull(mikeManagerId); + assertObjectIdEquals(mikeManagerId, juliusId); + + List mikeMentorIds = toAtlasObjectIds(mike.getRelationshipAttribute("mentors")); + assertEmpty(mikeMentorIds); + + List mikeMenteesId = toAtlasObjectIds(mike.getRelationshipAttribute("mentees")); + assertEmpty(mikeMenteesId); + + List mikeFriendsIds = toAtlasObjectIds(mike.getRelationshipAttribute("friends")); + assertNotNull(mikeFriendsIds); + assertEquals(mikeFriendsIds.size(), 2); + assertObjectIdsContains(mikeFriendsIds, maxId); + assertObjectIdsContains(mikeFriendsIds, johnId); + // Jane Manager validation AtlasObjectId janeDepartmentId = toAtlasObjectId(jane.getRelationshipAttribute("department")); assertNotNull(janeDepartmentId); @@ -189,8 +228,8 @@ public abstract class AtlasRelationshipStoreV1Test { AtlasObjectId janeManagerId = toAtlasObjectId(jane.getRelationshipAttribute("manager")); assertNull(janeManagerId); - AtlasObjectId janeMentorId = toAtlasObjectId(jane.getRelationshipAttribute("mentor")); - assertNull(janeMentorId); + List janeMentorIds = toAtlasObjectIds(jane.getRelationshipAttribute("mentors")); + assertEmpty(janeMentorIds); List janeMenteesId = toAtlasObjectIds(jane.getRelationshipAttribute("mentees")); assertEmpty(janeMenteesId); @@ -201,6 +240,13 @@ public abstract class AtlasRelationshipStoreV1Test { assertObjectIdsContains(janeSubordinateIds, maxId); assertObjectIdsContains(janeSubordinateIds, johnId); + List janeFriendsIds = toAtlasObjectIds(jane.getRelationshipAttribute("friends")); + assertEmpty(janeFriendsIds); + + AtlasObjectId janeSiblingId = toAtlasObjectId(jane.getRelationshipAttribute("sibling")); + assertNotNull(janeSiblingId); + assertObjectIdEquals(janeSiblingId, juliusId); + // Julius Manager validation AtlasObjectId juliusDepartmentId = toAtlasObjectId(julius.getRelationshipAttribute("department")); assertNotNull(juliusDepartmentId); @@ -209,16 +255,26 @@ public abstract class AtlasRelationshipStoreV1Test { AtlasObjectId juliusManagerId = toAtlasObjectId(julius.getRelationshipAttribute("manager")); assertNull(juliusManagerId); - AtlasObjectId juliusMentorId = toAtlasObjectId(julius.getRelationshipAttribute("mentor")); - assertNull(juliusMentorId); + List juliusMentorIds = toAtlasObjectIds(julius.getRelationshipAttribute("mentors")); + assertEmpty(juliusMentorIds); List juliusMenteesId = toAtlasObjectIds(julius.getRelationshipAttribute("mentees")); assertNotNull(juliusMenteesId); - assertEquals(juliusMenteesId.size(), 1); + assertEquals(juliusMenteesId.size(), 2); assertObjectIdsContains(juliusMenteesId, maxId); + assertObjectIdsContains(juliusMenteesId, johnId); List juliusSubordinateIds = toAtlasObjectIds(julius.getRelationshipAttribute("subordinates")); - assertEmpty(juliusSubordinateIds); + assertNotNull(juliusSubordinateIds); + assertEquals(juliusSubordinateIds.size(), 1); + assertObjectIdsContains(juliusSubordinateIds, mikeId); + + List juliusFriendsIds = toAtlasObjectIds(julius.getRelationshipAttribute("friends")); + assertEmpty(juliusFriendsIds); + + AtlasObjectId juliusSiblingId = toAtlasObjectId(julius.getRelationshipAttribute("sibling")); + assertNotNull(juliusSiblingId); + assertObjectIdEquals(juliusSiblingId, janeId); } @Test @@ -226,6 +282,8 @@ public abstract class AtlasRelationshipStoreV1Test { AtlasObjectId maxId = employeeNameIdMap.get("Max"); AtlasObjectId juliusId = employeeNameIdMap.get("Julius"); AtlasObjectId janeId = employeeNameIdMap.get("Jane"); + AtlasObjectId mikeId = employeeNameIdMap.get("Mike"); + AtlasObjectId johnId = employeeNameIdMap.get("John"); // Change Max's Employee.manager reference to Julius and apply the change as a partial update. // This should also update Julius to add Max to the inverse Manager.subordinates reference. @@ -249,9 +307,9 @@ public abstract class AtlasRelationshipStoreV1Test { AtlasEntity maxEntity = updatedEntities.getEntity(maxId.getGuid()); verifyRelationshipAttributeValue(maxEntity, "manager", juliusId.getGuid()); - // Max added to the subordinate list of Julius + // Max added to the subordinate list of Julius, existing subordinate is Mike AtlasEntity juliusEntity = updatedEntities.getEntity(juliusId.getGuid()); - verifyRelationshipAttributeList(juliusEntity, "subordinates", ImmutableList.of(maxId)); + verifyRelationshipAttributeList(juliusEntity, "subordinates", ImmutableList.of(maxId, mikeId)); // Max removed from the subordinate list of Julius AtlasEntity janeEntity = updatedEntities.getEntity(janeId.getGuid()); @@ -259,6 +317,48 @@ public abstract class AtlasRelationshipStoreV1Test { // Jane's subordinates list includes John and Max for soft delete // Jane's subordinates list includes only John for hard delete verifyRelationshipAttributeUpdate_NonComposite_OneToMany(janeEntity); + + // Remove Mike from Max's friends list + // Max's current friends: [Mike, John] + // Max's updated friends: [Julius, John] + maxEntityForUpdate = new AtlasEntity(EMPLOYEE_TYPE); + maxEntityForUpdate.setRelationshipAttribute("friends", ImmutableList.of(johnId, juliusId)); + + init(); + updateResponse = entityStore.updateByUniqueAttributes(employeeType, uniqAttributes , new AtlasEntityWithExtInfo(maxEntityForUpdate)); + + partialUpdatedEntities = updateResponse.getPartialUpdatedEntities(); + assertEquals(partialUpdatedEntities.size(), 3); + // 3 entities should have been updated: + // * Max added Julius and removed Mike from Employee.friends + // * Mike removed Max from Employee.friends + // * Julius added Max in Employee.friends + + updatedEntities = entityStore.getByIds(ImmutableList.of(maxId.getGuid(), mikeId.getGuid(), johnId.getGuid(), juliusId.getGuid())); + + maxEntity = updatedEntities.getEntity(maxId.getGuid()); + juliusEntity = updatedEntities.getEntity(juliusId.getGuid()); + AtlasEntity mikeEntity = updatedEntities.getEntity(mikeId.getGuid()); + AtlasEntity johnEntity = updatedEntities.getEntity(johnId.getGuid()); + + verifyRelationshipAttributeUpdate_ManyToMany_Friends(maxEntity, juliusEntity, mikeEntity, johnEntity); + + // Remove Julius from Jane's sibling and add Mike as new sibling + AtlasEntity juliusEntityForUpdate = new AtlasEntity(EMPLOYEE_TYPE); + juliusEntityForUpdate.setRelationshipAttribute("sibling", mikeId); + + init(); + updateResponse = entityStore.updateByUniqueAttributes(employeeType, Collections.singletonMap("name", "Julius") , new AtlasEntityWithExtInfo(juliusEntityForUpdate)); + partialUpdatedEntities = updateResponse.getPartialUpdatedEntities(); + assertEquals(partialUpdatedEntities.size(), 3); + + updatedEntities = entityStore.getByIds(ImmutableList.of(juliusId.getGuid(), janeId.getGuid(), mikeId.getGuid())); + + juliusEntity = updatedEntities.getEntity(juliusId.getGuid()); + janeEntity = updatedEntities.getEntity(janeId.getGuid()); + mikeEntity = updatedEntities.getEntity(mikeId.getGuid()); + + verifyRelationshipAttributeUpdate_OneToOne_Sibling(juliusEntity, janeEntity, mikeEntity); } @Test @@ -445,12 +545,16 @@ public abstract class AtlasRelationshipStoreV1Test { protected abstract void verifyRelationshipAttributeUpdate_NonComposite_ManyToOne(AtlasEntity a1, AtlasEntity a2, AtlasEntity a3, AtlasEntity b); - private static void assertObjectIdsContains(List objectIds, AtlasObjectId objectId) { + protected abstract void verifyRelationshipAttributeUpdate_ManyToMany_Friends(AtlasEntity e1, AtlasEntity e2, AtlasEntity e3, AtlasEntity e4) throws Exception; + + protected abstract void verifyRelationshipAttributeUpdate_OneToOne_Sibling(AtlasEntity e1, AtlasEntity e2, AtlasEntity e3) throws Exception; + + protected static void assertObjectIdsContains(List objectIds, AtlasObjectId objectId) { assertTrue(CollectionUtils.isNotEmpty(objectIds)); assertTrue(objectIds.contains(objectId)); } - private static void assertObjectIdEquals(AtlasObjectId objId1, AtlasObjectId objId2) { + protected static void assertObjectIdEquals(AtlasObjectId objId1, AtlasObjectId objId2) { assertTrue(objId1.equals(objId2)); } @@ -458,7 +562,7 @@ public abstract class AtlasRelationshipStoreV1Test { assertTrue(collection != null && collection.isEmpty()); } - private static List toAtlasObjectIds(Object object) { + protected static List toAtlasObjectIds(Object object) { List ret = new ArrayList<>(); if (object instanceof List) { @@ -477,7 +581,7 @@ public abstract class AtlasRelationshipStoreV1Test { return ret; } - private static AtlasObjectId toAtlasObjectId(Object object) { + protected static AtlasObjectId toAtlasObjectId(Object object) { if (object instanceof AtlasRelatedObjectId) { AtlasRelatedObjectId relatedObjectId = (AtlasRelatedObjectId) object; return new AtlasObjectId(relatedObjectId.getGuid(), relatedObjectId.getTypeName(), relatedObjectId.getUniqueAttributes());