atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dkan...@apache.org
Subject incubator-atlas git commit: ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor)
Date Wed, 01 Feb 2017 22:46:03 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 4367c4915 -> a810f1564


ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/a810f156
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/a810f156
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/a810f156

Branch: refs/heads/master
Commit: a810f15648e764d1e15f1e810718da8eba7758f6
Parents: 4367c49
Author: Dave Kantor <dkantor@us.ibm.com>
Authored: Wed Feb 1 17:45:57 2017 -0500
Committer: Dave Kantor <dkantor@us.ibm.com>
Committed: Wed Feb 1 17:45:57 2017 -0500

----------------------------------------------------------------------
 release-log.txt                                 |  1 +
 .../atlas/repository/graph/DeleteHandler.java   |  5 +-
 .../store/graph/v1/DeleteHandlerV1.java         |  5 +-
 ...hBackedMetadataRepositoryDeleteTestBase.java | 91 ++++++++++++++++++++
 .../GraphBackedRepositoryHardDeleteTest.java    |  8 ++
 .../GraphBackedRepositorySoftDeleteTest.java    |  8 ++
 6 files changed, 116 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index b0fc288..4459f9d 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements
for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor)
 ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb) 
 ATLAS-1513 updated AtlasEntityType with methods to get foreign-key references; added helper
methods in AtlasAttribute (mneethiraj via kevalbhatt)
 ATLAS-1502 added configuration to restrict entity-types editable via UI (Kalyanikashikar
via mneethiraj)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
index 4973a33..f0fef1f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
@@ -354,7 +354,10 @@ public abstract class DeleteHandler {
                                         attributeName);
                             }
 
-                            elements.remove(elementEdge.getId().toString());
+                            // Remove all occurrences of the edge ID from the list.
+                            // This prevents dangling edge IDs (i.e. edge IDs for deleted
edges)
+                            // from the remaining in the list if there are duplicates.
+                            elements.removeAll(Collections.singletonList(elementEdge.getId().toString()));
                             GraphHelper.setProperty(outVertex, propertyName, elements);
                             break;
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index 08361ea..a989f76 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -447,7 +447,10 @@ public abstract class DeleteHandlerV1 {
                             //but when column is deleted, table will not reference the deleted
column
                             LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge),
                                 attributeName);
-                            elements.remove(elementEdge.getId().toString());
+                            // Remove all occurrences of the edge ID from the list.
+                            // This prevents dangling edge IDs (i.e. edge IDs for deleted
edges)
+                            // from the remaining in the list if there are duplicates.
+                            elements.removeAll(Collections.singletonList(elementEdge.getId().toString()));
                             GraphHelper.setProperty(outVertex, propertyName, elements);
                             break;
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
index 9e850a9..4919b08 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
@@ -1048,6 +1048,97 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
             Arrays.asList(mapOwnerGuid, mapValueGuid, mapValueReferencerContainerGuid, mapValueReferencerGuid)));
     }
 
+    @Test
+    public void testDeleteMixOfExistentAndNonExistentEntities() throws Exception {
+        ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance();
+        ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance();
+        List<String> createEntitiesResult = repositoryService.createEntities(entity1,
entity2);
+        Assert.assertEquals(createEntitiesResult.size(), 2);
+        List<String> guids = Arrays.asList(createEntitiesResult.get(0), "non-existent-guid1",
"non-existent-guid2", createEntitiesResult.get(1));
+        EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids);
+        Assert.assertEquals(deleteEntitiesResult.getDeletedEntities().size(), 2);
+        Assert.assertTrue(deleteEntitiesResult.getDeletedEntities().containsAll(createEntitiesResult));
+    }
+
+    @Test
+    public void testDeleteMixOfNullAndNonNullGuids() throws Exception {
+        ITypedReferenceableInstance entity1 = compositeMapValueType.createInstance();
+        ITypedReferenceableInstance entity2 = compositeMapValueType.createInstance();
+        List<String> createEntitiesResult = repositoryService.createEntities(entity1,
entity2);
+        Assert.assertEquals(createEntitiesResult.size(), 2);
+        List<String> guids = Arrays.asList(createEntitiesResult.get(0), null, null,
createEntitiesResult.get(1));
+        EntityResult deleteEntitiesResult = repositoryService.deleteEntities(guids);
+        Assert.assertEquals(deleteEntitiesResult.getDeletedEntities().size(), 2);
+        Assert.assertTrue(deleteEntitiesResult.getDeletedEntities().containsAll(createEntitiesResult));
+    }
+
+    @Test
+    public void testDeleteCompositeEntityAndContainer() throws Exception {
+        Referenceable db = createDBEntity();
+        String dbId = createInstance(db);
+
+        Referenceable column = createColumnEntity();
+        String colId = createInstance(column);
+
+        Referenceable table1 = createTableEntity(dbId);
+        table1.set(COLUMNS_ATTR_NAME, Arrays.asList(new Id(colId, 0, COLUMN_TYPE)));
+        String table1Id = createInstance(table1);
+        Referenceable table2 = createTableEntity(dbId);
+        String table2Id = createInstance(table2);
+
+        // Delete the tables and column
+        AtlasClient.EntityResult entityResult = deleteEntities(table1Id, colId, table2Id);
+        Assert.assertEquals(entityResult.getDeletedEntities().size(), 3);
+        Assert.assertTrue(entityResult.getDeletedEntities().containsAll(Arrays.asList(colId,
table1Id, table2Id)));
+        assertEntityDeleted(table1Id);
+        assertEntityDeleted(colId);
+        assertEntityDeleted(table2Id);
+    }
+
+    @Test
+    public void testDeleteEntityWithDuplicateReferenceListElements() throws Exception {
+        // Create a table entity, with 2 composite column entities
+        Referenceable dbEntity = createDBEntity();
+        String dbGuid = createInstance(dbEntity);
+        Referenceable table1Entity = createTableEntity(dbGuid);
+        String tableName = TestUtils.randomString();
+        table1Entity.set(NAME, tableName);
+        Referenceable col1 = createColumnEntity();
+        col1.set(NAME, TestUtils.randomString());
+        Referenceable col2 = createColumnEntity();
+        col2.set(NAME, TestUtils.randomString());
+        // Populate columns reference list with duplicates.
+        table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col1, col2));
+        ClassType dataType = typeSystem.getDataType(ClassType.class, table1Entity.getTypeName());
+        ITypedReferenceableInstance instance = dataType.convert(table1Entity, Multiplicity.REQUIRED);
+        TestUtils.resetRequestContext();
+        List<String> result = repositoryService.createEntities(instance);
+        Assert.assertEquals(result.size(), 3);
+        ITypedReferenceableInstance entityDefinition = repositoryService.getEntityDefinition(TABLE_TYPE,
NAME, tableName);
+        String tableGuid = entityDefinition.getId()._getId();
+        Object attrValue = entityDefinition.get(COLUMNS_ATTR_NAME);
+        assertTrue(attrValue instanceof List);
+        List<ITypedReferenceableInstance> columns = (List<ITypedReferenceableInstance>)
attrValue;
+        Assert.assertEquals(columns.size(), 4);
+        TestUtils.resetRequestContext();
+        String columnGuid = columns.get(0).getId()._getId();
+
+        // Delete one of the columns.
+        EntityResult deleteResult = repositoryService.deleteEntities(Collections.singletonList(columnGuid));
+        Assert.assertEquals(deleteResult.getDeletedEntities().size(), 1);
+        Assert.assertTrue(deleteResult.getDeletedEntities().contains(columnGuid));
+        Assert.assertEquals(deleteResult.getUpdateEntities().size(), 1);
+        Assert.assertTrue(deleteResult.getUpdateEntities().contains(tableGuid));
+
+        // Verify the duplicate edge IDs were all removed from reference property list.
+        AtlasVertex tableVertex = GraphHelper.getInstance().getVertexForGUID(tableGuid);
+        String columnsPropertyName = GraphHelper.getQualifiedFieldName(dataType, COLUMNS_ATTR_NAME);
+        List columnsPropertyValue = tableVertex.getProperty(columnsPropertyName, List.class);
+        verifyTestDeleteEntityWithDuplicateReferenceListElements(columnsPropertyValue);
+    }
+
+    protected abstract void verifyTestDeleteEntityWithDuplicateReferenceListElements(List
columnsPropertyValue);
+
     private String createHrDeptGraph() throws Exception {
         ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
index 7929505..43b08c4 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
@@ -202,4 +202,12 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo
             // good
         }
     }
+
+    @Override
+    protected void verifyTestDeleteEntityWithDuplicateReferenceListElements(List columnsPropertyValue)
{
+
+        // With hard deletes enabled, verify that duplicate edge IDs for deleted edges
+        // were removed from the array property list.
+        Assert.assertEquals(columnsPropertyValue.size(), 2);
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a810f156/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java
b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java
index 93926ef..5ac7c8f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java
@@ -229,4 +229,12 @@ public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepo
         ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
         Assert.assertEquals(hrDept.getId().getState(), EntityState.DELETED);
     }
+
+    @Override
+    protected void verifyTestDeleteEntityWithDuplicateReferenceListElements(List columnsPropertyValue)
{
+
+        // With soft deletes enabled, verify that edge IDs for deleted edges
+        // were not removed from the array property list.
+        Assert.assertEquals(columnsPropertyValue.size(), 4);
+    }
 }


Mime
View raw message