atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suma...@apache.org
Subject [22/26] incubator-atlas git commit: ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)
Date Fri, 18 Dec 2015 10:44:22 GMT
ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ae1d636f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ae1d636f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ae1d636f

Branch: refs/heads/branch-0.6-incubating
Commit: ae1d636fbb48346f4cfc926be469a6bfc884f71e
Parents: 8cbb2c1
Author: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Authored: Thu Dec 17 21:56:03 2015 +0530
Committer: Suma Shivaprasad <sumasai.shivaprasad@gmail.com>
Committed: Thu Dec 17 21:56:03 2015 +0530

----------------------------------------------------------------------
 dashboard/public/css/lineage.css                |   1 +
 .../public/modules/details/detailsController.js |  10 +-
 .../public/modules/details/views/details.html   |   2 +-
 .../modules/lineage/lineage_ioController.js     | 131 ++++++-------
 release-log.txt                                 |   1 +
 ...kedMetadataRepositoryDeleteEntitiesTest.java | 189 +++++++++++++++++++
 6 files changed, 254 insertions(+), 80 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/css/lineage.css
----------------------------------------------------------------------
diff --git a/dashboard/public/css/lineage.css b/dashboard/public/css/lineage.css
index d700e34..180bebf 100644
--- a/dashboard/public/css/lineage.css
+++ b/dashboard/public/css/lineage.css
@@ -86,4 +86,5 @@ div.lineage {
 .alignLineage{
     text-align: center;
     margin-top: 100px;
+    font-size: 24px;
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/modules/details/detailsController.js
----------------------------------------------------------------------
diff --git a/dashboard/public/modules/details/detailsController.js b/dashboard/public/modules/details/detailsController.js
index c03824c..cc6bb2d 100644
--- a/dashboard/public/modules/details/detailsController.js
+++ b/dashboard/public/modules/details/detailsController.js
@@ -22,14 +22,14 @@ angular.module('dgc.details').controller('DetailsController', ['$window',
'$scop
 
         $scope.tableName = false;
         $scope.isTable = false;
-
+        $scope.isLineage = false;
+        
         DetailsResource.get({
             id: $stateParams.id
 
         }, function(data) {
             $scope.details = data;
             $scope.tableName = data.values.name;
-            $scope.isTable = (typeof data.typeName !== 'undefined' && data.typeName.toLowerCase().indexOf('table')
!== -1) ? true : false;
             $scope.onActivate('io');
             $scope.isTags = (typeof data.traits !== 'undefined' && typeof data.traits
=== 'object') ? true : false;
 
@@ -56,6 +56,9 @@ angular.module('dgc.details').controller('DetailsController', ['$window',
'$scop
             }
         });
 
+        $scope.$on('show_lineage', function() {
+            $scope.isLineage = true;
+        });
 
         $scope.isNumber = angular.isNumber;
         $scope.isObject = angular.isObject;
@@ -64,7 +67,8 @@ angular.module('dgc.details').controller('DetailsController', ['$window',
'$scop
         $scope.onActivate = function tabActivate(tabname) {
             $scope.$broadcast('render-lineage', {
                 type: tabname,
-                tableName: $scope.tableName
+                tableName: $scope.tableName,
+                guid : $stateParams.id
             });
         };
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/modules/details/views/details.html
----------------------------------------------------------------------
diff --git a/dashboard/public/modules/details/views/details.html b/dashboard/public/modules/details/views/details.html
index 0878308..c84aeb2 100644
--- a/dashboard/public/modules/details/views/details.html
+++ b/dashboard/public/modules/details/views/details.html
@@ -28,7 +28,7 @@
             <h4 ng-if="details.values && details.values.name && details.values.name
!= ''">
             <b>Name:</b> <span class="black">{{details.values.name}}</span></h2>
             <h4 ng-if="details.values && details.values.description &&
details.values.description != ''"><b>Description:</b> <span class="black">{{details.values.description}}</span></h4>
-            <h4 data-ng-show="isTable" data-disable="!tableName" data-select="onActivate('io')">
+            <h4 data-disable="!tableName" data-select="onActivate('io')" id="lineageGraph"
class="hide">
                 <span class="lineage">Lineage</span> 
                 <ng-include data-table-type="io" src="'/modules/lineage/views/lineage_io.html'"/>
             </h4>

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/dashboard/public/modules/lineage/lineage_ioController.js
----------------------------------------------------------------------
diff --git a/dashboard/public/modules/lineage/lineage_ioController.js b/dashboard/public/modules/lineage/lineage_ioController.js
index e7b2734..bb8fcec 100644
--- a/dashboard/public/modules/lineage/lineage_ioController.js
+++ b/dashboard/public/modules/lineage/lineage_ioController.js
@@ -39,51 +39,64 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element',
'$
             LineageResource.get({
                 tableName: tableData.tableName,
                 type: 'outputs'
-            }, function lineageSuccess(response1) {
-
-                LineageResource.get({
-                    tableName: tableData.tableName,
-                    type: 'inputs'
-                }, function lineageSuccess(response) {
-                    response.results.values.edges = inVertObj(response.results.values.edges);
-
-                    angular.forEach(response.results.values.edges, function(value, key) {
-                        angular.forEach(response1.results.values.edges, function(value1,
key1) {
-                            if (key === key1) {
-                                var array1 = value;
-                                angular.forEach(value1, function(value2) {
-                                    array1.push(value2);
+            }).$promise.then(
+                function lineageSuccess(response1) {
+                  //  $scope.$emit('show_lineage');
+                  $('#lineageGraph').removeClass('hide');
+                    LineageResource.get({
+                        tableName: tableData.tableName,
+                        type: 'inputs'
+                    }).$promise.then(
+                        //success
+                        function lineageSuccess(response) {
+                            if (response && response.results) {
+                                response.results.values.edges = inVertObj(response.results.values.edges);
+
+                                angular.forEach(response.results.values.edges, function(value,
key) {
+                                    angular.forEach(response1.results.values.edges, function(value1,
key1) {
+                                        if (key === key1) {
+                                            var array1 = value;
+                                            angular.forEach(value1, function(value2) {
+                                                array1.push(value2);
+                                            });
+                                            response.results.values.edges[key] = array1;
+                                            response1.results.values.edges[key] = array1;
+                                        }
+                                    });
                                 });
-                                response.results.values.edges[key] = array1;
-                                response1.results.values.edges[key] = array1;
-                            }
-                        });
-                    });
 
-                    angular.extend(response.results.values.edges, response1.results.values.edges);
-                    angular.extend(response.results.values.vertices, response1.results.values.vertices);
+                                angular.extend(response.results.values.edges, response1.results.values.edges);
+                                angular.extend(response.results.values.vertices, response1.results.values.vertices);
 
-                    if (!_.isEmpty(response.results.values.vertices)) {
-                        loadProcess(response.results.values.edges, response.results.values.vertices)
-                            .then(function(res) {
-                                guidsList = res;
+                                if (!_.isEmpty(response.results.values.vertices)) {
+                                    loadProcess(response.results.values.edges, response.results.values.vertices)
+                                        .then(function(res) {
+                                            guidsList = res;
 
-                                $scope.lineageData = transformData(response.results);
+                                            $scope.lineageData = transformData(response.results);
 
-                                if (callRender) {
-                                    render();
+                                            if (callRender) {
+                                                render();
+                                            }
+                                        });
+                                } else {
+                                    $scope.requested = false;
                                 }
-                            });
-                    } else {
-                        $scope.requested = false;
-                    }
-                });
-
-            });
-
+                            } else {
+                                $scope.requested = false;
+                            }
+                        },
+                        function() {
+                            $scope.requested = false;
+                        }
+                    );
+                },
+                function() {
+                    $scope.requested = false;
+                }
+            );
         }
 
-
         function loadProcess(edges, vertices) {
 
             var urlCalls = [];
@@ -141,6 +154,7 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element',
'$
                     render();
                 }
             }
+            $scope.guid = lineageData.guid;
         });
 
         function transformData(metaData) {
@@ -157,11 +171,11 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element',
'$
                     var loadProcess = getLoadProcessTypes(guid);
                     if (typeof loadProcess !== "undefined") {
                         name = loadProcess.name;
-                        type = loadProcess.typeName;
+                        type = 'edges';
                         tip = loadProcess.tip;
                     } else {
                         name = 'Load Process';
-                        type = 'Load Process';
+                        type = 'edges';
                     }
                 }
                 var vertex = {
@@ -433,29 +447,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element',
'$
                 zoomListener.scale(scale);
                 zoomListener.translate([x, y]);
             }
-
-            // Toggle children function
-
-            // function toggleChildren(d) {
-            //     if (d.children) {
-            //         d._children = d.children;
-            //         d.children = null;
-            //     } else if (d._children) {
-            //         d.children = d._children;
-            //         d._children = null;
-            //     }
-            //     return d;
-            // }
-
-            // Toggle children on click.
-
-            // function click(d) {
-            //     if (d3.event.defaultPrevented) return; // click suppressed
-            //     d = toggleChildren(d);
-            //     update(d);
-            //     //centerNode(d);
-            // }
-
             //arrow
             baseSvg.append("svg:defs")
                 .append("svg:marker")
@@ -568,10 +559,10 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element',
'$
                 nodeEnter.append("image")
                     .attr("class", "nodeImage")
                     .attr("xlink:href", function(d) {
-                        return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png';
+                        return (d.type && d.type !== '' && d.type.toLowerCase().indexOf('edges')
!== -1) ? '../img/process.png' : '../img/tableicon.png';
                     })
                     .on('mouseover', function(d) {
-                        if (d.type === 'LoadProcess' || 'Table') {
+                        if (d.type === 'edges' || 'Table') {
                             tooltip.show(d);
                         }
                     })
@@ -623,18 +614,6 @@ angular.module('dgc.lineage').controller('Lineage_ioController', ['$element',
'$
                         return nameDis;
                     });
 
-                // Change the circle fill depending on whether it has children and is collapsed
-                // Change the circle fill depending on whether it has children and is collapsed
-                node.select("image.nodeImage")
-                    .attr("r", 4.5)
-                    .attr("xlink:href", function(d) {
-                        if (d._children) {
-                            return d.type === 'Table' ? '../img/tableicon1.png' : '../img/process1.png';
-                        }
-                        return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png';
-                    });
-
-
                 // Transition nodes to their new position.
                 var nodeUpdate = node.transition()
                     .duration(duration)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index ce3337d..7d45a7b 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
 ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
 
 ALL CHANGES:
+ATLAS-385 Support for Lineage for entities with SuperType as DataSet (anilsg via sumasai)
 ATLAS-342 Atlas is sending an ENTITY_CREATE event to the ATLAS_ENTITIES topic even if the
entity exists already (shwethags)
 ATLAS-386 Handle hive rename Table (shwethags)
 ATLAS-374 Doc: Create a wiki for documenting fault tolerance and HA options for Atlas data
(yhemanth via sumasai)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ae1d636f/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
new file mode 100644
index 0000000..e63f6d3
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.inject.Inject;
+
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Test for GraphBackedMetadataRepository.deleteEntities
+ *
+ * Guice loads the dependencies and injects the necessary objects
+ *
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
+
+    @Inject
+    private GraphProvider<TitanGraph> graphProvider;
+
+    @Inject
+    private GraphBackedMetadataRepository repositoryService;
+
+    @Inject
+    private GraphBackedDiscoveryService discoveryService;
+
+    private TypeSystem typeSystem;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        typeSystem = TypeSystem.getInstance();
+        typeSystem.reset();
+
+        new GraphBackedSearchIndexer(graphProvider);
+
+        TestUtils.defineDeptEmployeeTypes(typeSystem);
+        TestUtils.createHiveTypes(typeSystem);
+    }
+
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        TypeSystem.getInstance().reset();
+        try {
+            graphProvider.get().shutdown();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        try {
+            TitanCleanup.clear(graphProvider.get());
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+
+
+    @Test
+    public void testDeleteEntities() throws Exception {
+        String hrDeptGuid = createHrDeptGraph();
+
+        ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
+        Object refValue = hrDept.get("employees");
+        Assert.assertTrue(refValue instanceof List);
+        List<Object> employees = (List<Object>)refValue;
+        Assert.assertEquals(employees.size(), 4);
+        List<String> employeeGuids = new ArrayList<String>(4);
+        for (Object listValue : employees) {
+            Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
+            ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
+            employeeGuids.add(employee.getId()._getId());
+        }
+        
+        // There should be 4 vertices for Address structs (one for each Person.address attribute
value).
+        int vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
+        Assert.assertEquals(vertexCount, 4);
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
+        Assert.assertEquals(vertexCount, 1);
+        
+        List<String> deletedEntities = repositoryService.deleteEntities(hrDeptGuid);
+        Assert.assertTrue(deletedEntities.contains(hrDeptGuid));
+        
+        // Verify Department entity and its contained Person entities were deleted.
+        verifyEntityDoesNotExist(hrDeptGuid);
+        for (String employeeGuid : employeeGuids) {
+            verifyEntityDoesNotExist(employeeGuid);
+        }
+        // Verify all Person.address struct vertices were removed.
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
+        Assert.assertEquals(vertexCount, 0);
+        
+        // Verify all SecurityClearance trait vertices were removed.
+        vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "SecurityClearance");
+        Assert.assertEquals(vertexCount, 0);
+    }
+
+    @Test(dependsOnMethods = "testDeleteEntities")
+    public void testDeleteContainedEntity() throws Exception {
+        String hrDeptGuid = createHrDeptGraph();
+        ITypedReferenceableInstance hrDept = repositoryService.getEntityDefinition(hrDeptGuid);
+        Object refValue = hrDept.get("employees");
+        Assert.assertTrue(refValue instanceof List);
+        List<Object> employees = (List<Object>)refValue;
+        Assert.assertEquals(employees.size(), 4);
+        Object listValue = employees.get(2);
+        Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
+        ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
+        String employeeGuid = employee.getId()._getId();
+        
+        List<String> deletedEntities = repositoryService.deleteEntities(employeeGuid);
+        Assert.assertTrue(deletedEntities.contains(employeeGuid));
+        verifyEntityDoesNotExist(employeeGuid);
+        
+    }
+
+    private String createHrDeptGraph() throws Exception {
+        Referenceable deptEg1 = TestUtils.createDeptEg1(typeSystem);
+        ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
+        ITypedReferenceableInstance hrDept2 = deptType.convert(deptEg1, Multiplicity.REQUIRED);
+
+        List<String> guids = repositoryService.createEntities(hrDept2);
+        Assert.assertNotNull(guids);
+        Assert.assertEquals(guids.size(), 5);
+        List<String> entityList = repositoryService.getEntityList("Department");
+        Assert.assertNotNull(entityList);
+        Assert.assertEquals(entityList.size(), 1);
+        String hrDeptGuid = entityList.get(0);
+        return hrDeptGuid;
+    }
+    
+    private int countVertices(String propertyName, Object value) {
+        Iterable<Vertex> vertices = graphProvider.get().getVertices(propertyName, value);
+        int vertexCount = 0;
+        for (Vertex vertex : vertices) {
+            vertexCount++;
+        }
+        return vertexCount;
+    }
+    
+    private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
+
+        try {
+            repositoryService.getEntityDefinition(hrDeptGuid);
+            Assert.fail("EntityNotFoundException was expected but none thrown");
+        }
+        catch(EntityNotFoundException e) {
+            // good
+        }
+    }
+
+}


Mime
View raw message