atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jnhagelb...@apache.org
Subject incubator-atlas git commit: ATLAS-1510: Consolidate/batch calls to GraphBackedTypeStore.findVertex()
Date Fri, 03 Feb 2017 17:00:36 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 09089e09f -> 02cf8c488


ATLAS-1510: Consolidate/batch calls to GraphBackedTypeStore.findVertex()


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/02cf8c48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/02cf8c48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/02cf8c48

Branch: refs/heads/master
Commit: 02cf8c4884eb346f7b993bd46d6e789bd73f006d
Parents: 09089e0
Author: Jeff Hagelberg <jnhagelberg@us.ibm.com>
Authored: Fri Feb 3 12:00:09 2017 -0500
Committer: Jeff Hagelberg <jnhagelberg@us.ibm.com>
Committed: Fri Feb 3 12:00:09 2017 -0500

----------------------------------------------------------------------
 release-log.txt                                 |   1 +
 .../atlas/repository/graph/GraphHelper.java     |   2 +-
 .../typestore/GraphBackedTypeStore.java         | 156 +++++++++++++------
 .../typestore/TypePersistenceVisitor.java       | 116 ++++++++++++++
 .../repository/typestore/TypeVertexFinder.java  | 103 ++++++++++++
 .../repository/typestore/TypeVertexInfo.java    |  94 +++++++++++
 .../atlas/repository/typestore/TypeVisitor.java |  96 ++++++++++++
 .../typestore/GraphBackedTypeStoreTest.java     |   6 +-
 8 files changed, 520 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 991557b..e6272fb 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements
for al
 ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
 
 ALL CHANGES:
+ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb)
 ATLAS-1388 Cache entities that are created/updated (jnhagelb)
 ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb)
 ATLAS-1517: updated hive_model to include schema related attributes (sarath.kum4r@gmail.com
via mneethiraj)

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index ce8d4c7..89e978d 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -538,7 +538,7 @@ public final class GraphHelper {
      *
      *  @return propertyValue to AtlasVertex map with the result.
      */
-    private Map<String, AtlasVertex> getVerticesForPropertyValues(String property,
List<String> values)
+    public Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String>
values)
             throws RepositoryException {
 
         if(values.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
old mode 100755
new mode 100644
index c1c2a94..39e7a31
--- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
@@ -21,14 +21,17 @@ package org.apache.atlas.repository.typestore;
 import static org.apache.atlas.repository.graph.GraphHelper.setProperty;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.GraphTransaction;
 import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
 import org.apache.atlas.repository.graph.AtlasGraphProvider;
 import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -57,8 +60,10 @@ import org.codehaus.jettison.json.JSONException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 
@@ -82,25 +87,37 @@ public class GraphBackedTypeStore implements ITypeStore {
     @Override
     @GraphTransaction
     public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws
AtlasException {
+
+        //Pre-create the vertices that are needed for the types.  This allows us to execute
+        //one query to determine all of the vertices that already exist.
+        Map<String, AtlasVertex> typeVertices = getOrCreateTypeVertices(typeSystem,
typeNames);
+
+        //Complete the storage process by adding properties and edges to the vertices
+        //that were created.
+        TypePersistenceVisitor visitor = new TypePersistenceVisitor(this, typeVertices, typeSystem);
+        processTypes(typeNames, typeSystem, visitor);
+    }
+
+    private void processTypes(ImmutableList<String> typeNames, TypeSystem typeSystem,
TypeVisitor visitor) throws AtlasException {
         for (String typeName : typeNames) {
             IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
             LOG.debug("Processing {}.{}.{} in type store", dataType.getTypeCategory(), dataType.getName(),
dataType.getDescription());
             switch (dataType.getTypeCategory()) {
             case ENUM:
-                storeInGraph((EnumType) dataType);
+                visitor.visitEnumeration((EnumType)dataType);
                 break;
 
             case STRUCT:
                 StructType structType = (StructType) dataType;
-                storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(),
dataType.getDescription(),
-                        ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of());
+                processType(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(),
+                        ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of(),
visitor);
                 break;
 
             case TRAIT:
             case CLASS:
                 HierarchicalType type = (HierarchicalType) dataType;
-                storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(),
type.getDescription(), type.immediateAttrs,
-                        type.superTypes);
+                processType(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(),
type.immediateAttrs,
+                        type.superTypes, visitor);
                 break;
 
             default:    //Ignore primitive/collection types as they are covered under references
@@ -109,63 +126,70 @@ public class GraphBackedTypeStore implements ITypeStore {
         }
     }
 
-    private void storeInGraph(EnumType dataType) {
-        AtlasVertex AtlasVertex = createVertex(dataType.getTypeCategory(), dataType.getName(),
dataType.getDescription());
-        List<String> values = new ArrayList<>(dataType.values().size());
-        for (EnumValue enumValue : dataType.values()) {
-            String key = getPropertyKey(dataType.getName(), enumValue.value);
-            setProperty(AtlasVertex, key, enumValue.ordinal);
-            values.add(enumValue.value);
+    private Map<String, AtlasVertex> getOrCreateTypeVertices(TypeSystem typeSystem,
ImmutableList<String> typeNames) throws AtlasException {
+
+        //examine the types to determine what type vertices are needed
+        TypeVertexFinder vertexFinder = new TypeVertexFinder(typeSystem);
+        processTypes(typeNames, typeSystem, vertexFinder);
+        List<TypeVertexInfo> typeVerticesNeeded = vertexFinder.getVerticesToCreate();
+
+        //find or create the type vertices
+        List<AtlasVertex> vertices = createVertices(typeVerticesNeeded);
+
+        //Create a type name->AtlasVertex map with the result
+        Map<String, AtlasVertex> result = new HashMap<String,AtlasVertex>(typeVerticesNeeded.size());
+        for(int i = 0 ; i < typeVerticesNeeded.size(); i++) {
+            TypeVertexInfo createdVertexInfo = typeVerticesNeeded.get(i);
+            AtlasVertex createdVertex = vertices.get(i);
+            result.put(createdVertexInfo.getTypeName(), createdVertex);
         }
-        setProperty(AtlasVertex, getPropertyKey(dataType.getName()), values);
+        return result;
+
     }
 
-    private String getPropertyKey(String name) {
+
+    static String getPropertyKey(String name) {
         return PROPERTY_PREFIX + name;
     }
 
-    private String getPropertyKey(String parent, String child) {
+    static String getPropertyKey(String parent, String child) {
         return PROPERTY_PREFIX + parent + "." + child;
     }
 
-    String getEdgeLabel(String parent, String child) {
+    static String getEdgeLabel(String parent, String child) {
         return PROPERTY_PREFIX + "edge." + parent + "." + child;
     }
 
-    private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String
typeName, String typeDescription,
-            ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes)
throws AtlasException {
-        AtlasVertex vertex = createVertex(category, typeName, typeDescription);
+    private void processType(TypeSystem typeSystem, DataTypes.TypeCategory category, String
typeName, String typeDescription,
+            ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes,
TypeVisitor visitor) throws AtlasException {
+
+        visitor.visitDataType(category, typeName, typeDescription);
+
         List<String> attrNames = new ArrayList<>();
         if (attributes != null) {
             for (AttributeInfo attribute : attributes) {
-                String propertyKey = getPropertyKey(typeName, attribute.name);
-                try {
-                    setProperty(vertex, propertyKey, attribute.toJson());
-                } catch (JSONException e) {
-                    throw new StorageException(typeName, e);
-                }
+                visitor.visitAttribute(typeName, attribute);
                 attrNames.add(attribute.name);
-                addReferencesForAttribute(typeSystem, vertex, attribute);
+                processsAttribute(typeSystem,  typeName, attribute, visitor);
             }
         }
-        setProperty(vertex, getPropertyKey(typeName), attrNames);
+        visitor.visitAttributeNames(typeName, attrNames);
 
         //Add edges for hierarchy
         if (superTypes != null) {
             for (String superTypeName : superTypes) {
-                HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class,
superTypeName);
-                AtlasVertex superVertex = createVertex(superType.getTypeCategory(), superTypeName,
superType.getDescription());
-                graphHelper.getOrCreateEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
+                visitor.visitSuperType(typeName, superTypeName);
             }
         }
     }
 
-    private void addReferencesForAttribute(TypeSystem typeSystem, AtlasVertex vertex, AttributeInfo
attribute)
+    private void processsAttribute(TypeSystem typeSystem, String typeName, AttributeInfo
attribute, TypeVisitor visitor)
             throws AtlasException {
+
         ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
         List<IDataType> attrDataTypes = new ArrayList<>();
         IDataType attrDataType = attribute.dataType();
-        String vertexTypeName = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPENAME_PROPERTY_KEY,
String.class);
+
 
         switch (attrDataType.getTypeCategory()) {
         case ARRAY:
@@ -200,11 +224,10 @@ public class GraphBackedTypeStore implements ITypeStore {
                     "Attribute cannot reference instances of type : " + attrDataType.getTypeCategory());
         }
 
+
         for (IDataType attrType : attrDataTypes) {
             if (!coreTypes.contains(attrType.getName())) {
-                AtlasVertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(),
attrType.getDescription());
-                String label = getEdgeLabel(vertexTypeName, attribute.name);
-                graphHelper.getOrCreateEdge(vertex, attrVertex, label);
+                visitor.visitAttributeDataType(typeName, attribute, attrType);
             }
         }
     }
@@ -328,23 +351,54 @@ public class GraphBackedTypeStore implements ITypeStore {
         return vertex;
     }
 
-    private AtlasVertex createVertex(DataTypes.TypeCategory category, String typeName, String
typeDescription) {
-        AtlasVertex vertex = findVertex(category, typeName);
-        if (vertex == null) {
-            LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
-            vertex = graph.addVertex();
-            setProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark
as type AtlasVertex
-            setProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category);
-            setProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName);
-        }
-        if (typeDescription != null) {
-            String oldDescription = getPropertyKey(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
-            if (!typeDescription.equals(oldDescription)) {
-                setProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDescription);
+  //package-private for testing
+   Map<String, AtlasVertex> findVertices(List<String> typeNames) throws RepositoryException
{
+        LOG.debug("Finding vertices for {}", typeNames.toString());
+        Map<String, AtlasVertex> foundVertices = graphHelper.getVerticesForPropertyValues(Constants.TYPENAME_PROPERTY_KEY,
typeNames);
+        return foundVertices;
+
+    }
+
+
+    /**
+     * Finds or creates type vertices with the information specified.
+     *
+     * @param infoList
+     * @return list with the vertices corresponding to the types in the list.
+     * @throws AtlasException
+     */
+    private List<AtlasVertex> createVertices(List<TypeVertexInfo> infoList) throws
AtlasException {
+
+        List<AtlasVertex> result = new ArrayList<>(infoList.size());
+        List<String> typeNames = Lists.transform(infoList, new Function<TypeVertexInfo,String>()
{
+
+            @Override
+            public String apply(TypeVertexInfo input) {
+                return input.getTypeName();
+            }
+        });
+        Map<String, AtlasVertex> vertices = findVertices(typeNames);
+
+        for(TypeVertexInfo info : infoList) {
+            AtlasVertex vertex = vertices.get(info.getTypeName());
+            if (! GraphHelper.elementExists(vertex)) {
+                LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, info.getTypeName());
+                vertex = graph.addVertex();
+                setProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); //
Mark as type AtlasVertex
+                setProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, info.getCategory());
+                setProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, info.getTypeName());
             }
-        } else {
-            LOG.debug(" type description is null ");
+            String newDescription = info.getTypeDescription();
+            if (newDescription != null) {
+                String oldDescription = getPropertyKey(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
+                if (!newDescription.equals(oldDescription)) {
+                    setProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, newDescription);
+                }
+            } else {
+                LOG.debug(" type description is null ");
+            }
+            result.add(vertex);
         }
-        return vertex;
+        return result;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
b/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
new file mode 100644
index 0000000..bfb1bfc
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import static org.apache.atlas.repository.graph.GraphHelper.setProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.codehaus.jettison.json.JSONException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TypeVisitor implementation that completes the type storage process by
+ * adding the required properties and edges to the type vertices
+ * that were created.
+ */
+public class TypePersistenceVisitor implements TypeVisitor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TypePersistenceVisitor.class);
+    private static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+    private final GraphBackedTypeStore typeStore_;
+    private final Map<String,AtlasVertex> typeVertices;
+    private final TypeSystem typeSystem;
+
+    /**
+     * @param graphBackedTypeStore
+     */
+    public TypePersistenceVisitor(GraphBackedTypeStore graphBackedTypeStore, Map<String,AtlasVertex>
typeVertices, TypeSystem typeSystem) {
+        typeStore_ = graphBackedTypeStore;
+        this.typeVertices = typeVertices;
+        this.typeSystem = typeSystem;
+    }
+
+    @Override
+    public void visitEnumeration(EnumType dataType) throws AtlasException {
+        AtlasVertex vertex = typeVertices.get(dataType.getName());
+        List<String> values = new ArrayList<>(dataType.values().size());
+        for (EnumValue enumValue : dataType.values()) {
+            String key = GraphBackedTypeStore.getPropertyKey(dataType.getName(), enumValue.value);
+            setProperty(vertex, key, enumValue.ordinal);
+            values.add(enumValue.value);
+        }
+        setProperty(vertex, GraphBackedTypeStore.getPropertyKey(dataType.getName()), values);
+
+    }
+    @Override
+    public void visitAttributeDataType(String typeName, AttributeInfo attribute, IDataType
attrType) throws AtlasException {
+        AtlasVertex vertex = typeVertices.get(typeName);
+        String vertexTypeName = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPENAME_PROPERTY_KEY,
String.class);
+        AtlasVertex attrVertex = typeVertices.get(attrType.getName());
+        String label = GraphBackedTypeStore.getEdgeLabel(vertexTypeName, attribute.name);
+        graphHelper.getOrCreateEdge(vertex, attrVertex, label);
+    }
+    @Override
+    public void visitSuperType(String typeName, String superTypeName) throws AtlasException
{
+        AtlasVertex vertex = typeVertices.get(typeName);
+        HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
+        AtlasVertex superVertex = typeVertices.get(superTypeName);
+        graphHelper.getOrCreateEdge(vertex, superVertex, GraphBackedTypeStore.SUPERTYPE_EDGE_LABEL);
+    }
+
+    @Override
+    public void visitAttributeNames(String typeName, List<String> attrNames) throws
AtlasException {
+        AtlasVertex vertex = typeVertices.get(typeName);
+        setProperty(vertex, GraphBackedTypeStore.getPropertyKey(typeName), attrNames);
+
+    }
+
+    @Override
+    public void visitAttribute(String typeName, AttributeInfo attribute) throws AtlasException
{
+        AtlasVertex vertex = typeVertices.get(typeName);
+        String propertyKey = GraphBackedTypeStore.getPropertyKey(typeName, attribute.name);
+        try {
+            setProperty(vertex, propertyKey, attribute.toJson());
+        } catch (JSONException e) {
+            throw new StorageException(typeName, e);
+        }
+    }
+
+    @Override
+    public void visitDataType(TypeCategory category, String typeName, String typeDescription)
{
+        //nothing to do
+
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
new file mode 100644
index 0000000..8b38152
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+
+/**
+ * TypeVisitor implementation that builds up a list of type vertices
+ * that need to be created for the types that are being stored.
+ *
+ */
+public class TypeVertexFinder implements TypeVisitor {
+
+    private final List<TypeVertexInfo> toCreate = new ArrayList<TypeVertexInfo>();
+    private final Set<String> typesIncluded = new HashSet<String>();
+    private final TypeSystem typeSystem;
+
+    public TypeVertexFinder(TypeSystem ts) {
+        typeSystem = ts;
+    }
+
+
+    @Override
+    public void visitEnumeration(EnumType dataType) {
+        visitDataType(dataType);
+    }
+
+    private void addTypeIfNeeded(TypeVertexInfo info) {
+        if(! typesIncluded.contains(info.getTypeName())) {
+            toCreate.add(info);
+            typesIncluded.add(info.getTypeName());
+        }
+    }
+
+    @Override
+    public void visitAttributeDataType(String typeName, AttributeInfo sourceAttr, IDataType
attrType) throws AtlasException {
+        visitDataType(attrType);
+    }
+
+    @Override
+    public void visitSuperType(String typeName, String superTypeName) throws AtlasException
{
+        HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
+        visitDataType(superType);
+    }
+
+    @Override
+    public void visitAttributeNames(String typeName, List<String> attrNames) throws
AtlasException {
+       //nothing to do
+
+    }
+
+    @Override
+    public void visitAttribute(String typeName, AttributeInfo attribute) throws StorageException,
AtlasException {
+        //nothing to do
+    }
+
+
+    private void visitDataType(IDataType dataType) {
+        TypeVertexInfo info = null;
+        info = new TypeVertexInfo(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
+        addTypeIfNeeded(info);
+
+    }
+
+
+    public List<TypeVertexInfo> getVerticesToCreate() {
+        return toCreate;
+    }
+
+    @Override
+    public void visitDataType(TypeCategory category, String typeName, String typeDescription)
{
+        TypeVertexInfo info = new TypeVertexInfo(category, typeName, typeDescription);
+        addTypeIfNeeded(info);
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
new file mode 100644
index 0000000..32a9a19
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import java.util.Objects;
+
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+
+/**
+ * Records the information needed to create a particular type vertex.
+ */
+public class TypeVertexInfo {
+
+    private DataTypes.TypeCategory category;
+    private String typeName;
+    private String typeDescription;
+
+    public TypeVertexInfo(TypeCategory category, String typeName, String typeDescription)
{
+        super();
+        this.category = category;
+        this.typeName = typeName;
+        this.typeDescription = typeDescription;
+    }
+
+    public DataTypes.TypeCategory getCategory() {
+        return category;
+    }
+
+    public void setCategory(DataTypes.TypeCategory category) {
+        this.category = category;
+    }
+
+    public String getTypeName() {
+        return typeName;
+    }
+
+    public void setTypeName(String typeName) {
+        this.typeName = typeName;
+    }
+
+    public String getTypeDescription() {
+        return typeDescription;
+    }
+
+    public void setTypeDescription(String typeDescription) {
+        this.typeDescription = typeDescription;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(category, typeName);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+
+        if (this == obj) {
+            return true;
+        }
+
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+
+        TypeVertexInfo other = (TypeVertexInfo)obj;
+        if(! Objects.equals(category, other.category)) {
+            return false;
+        }
+
+        if(! Objects.equals(typeName, other.typeName)) {
+            return false;
+        }
+
+        return true;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
new file mode 100644
index 0000000..a6e353c
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.IDataType;
+
+/**
+ * Callback mechanism used when storing types.  As {@link GraphBackedTypeStore} traverses
+ * through the types being persisted, these methods are called with the information that
+ * it finds.
+ */
+public interface TypeVisitor {
+
+    /**
+     * Called when an enumeration type is found
+     * @param type
+     * @throws AtlasException
+     */
+    void visitEnumeration(EnumType type) throws AtlasException;
+
+    /**
+     * Called with a data type that is associated with a given attribute.  There can
+     * be more than one.  For example, map types have both a key and a value type.
+     * This is called once for each type.  This is called once for each datatype
+     * associated with the given attribute.
+     *
+     * @param typeName The name of the type being processed.
+     * @param sourceAttr The attribute in that type that we are processing.
+     * @param attrType A dataType associated with that attribute.
+     * @throws AtlasException
+     */
+    void visitAttributeDataType(String typeName, AttributeInfo sourceAttr, IDataType attrType)
throws AtlasException;
+
+    /**
+     * Called when a super type is found.  It is called once for each superType.
+     *
+     * @param typeName The type being processed.
+     * @param superType The name of the super type that was found.
+     * @throws RepositoryException
+     * @throws AtlasException
+     */
+    void visitSuperType(String typeName, String superType) throws RepositoryException, AtlasException;
+
+    /**
+     * Called with the list of immediate attribute names that were found for the given type.
 It
+     * is called once per type.
+     *
+     * @param typeName The name of the type that is being processed.
+     * @param attrNames The names of all of the immediate attributes in the type.
+     * @throws AtlasException
+     */
+    void visitAttributeNames(String typeName, List<String> attrNames) throws AtlasException;
+
+    /**
+     * Called once for each immediate attribute in a type.
+     * @param typeName The name of the type that is being procesed
+     * @param attribute The immediate attribute that was found
+     *
+     * @throws StorageException
+     * @throws AtlasException
+     */
+    void visitAttribute(String typeName, AttributeInfo attribute) throws StorageException,
AtlasException;
+
+    /**
+     * Called once for each struct, class, and trait type that was found.  It is
+     * called when we start processing that type.
+     *
+     * @param category The category of the type
+     * @param typeName The name of the type
+     * @param typeDescription The description of the type.
+     */
+    void visitDataType(TypeCategory category, String typeName, String typeDescription);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
index 6b83aa3..265b316 100755
--- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
@@ -23,6 +23,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAt
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
 import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
 
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -32,6 +33,7 @@ import javax.inject.Inject;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.TestUtils;
+import org.apache.atlas.repository.RepositoryException;
 import org.apache.atlas.repository.graph.AtlasGraphProvider;
 import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -200,11 +202,11 @@ public class GraphBackedTypeStoreTest {
         verifyEdges();
     }
 
-    private void verifyEdges() {
+    private void verifyEdges() throws RepositoryException {
         // ATLAS-474: verify that type update did not write duplicate edges to the type store.
         if (typeStore instanceof GraphBackedTypeStore) {
             GraphBackedTypeStore gbTypeStore = (GraphBackedTypeStore) typeStore;
-            AtlasVertex typeVertex = gbTypeStore.findVertex(TypeCategory.CLASS, "Department");
+            AtlasVertex typeVertex = gbTypeStore.findVertices(Collections.singletonList("Department")).get("Department");
             int edgeCount = countOutgoingEdges(typeVertex, gbTypeStore.getEdgeLabel("Department",
"employees"));
             Assert.assertEquals(edgeCount, 1, "Should only be 1 edge for employees attribute
on Department type AtlasVertex");
         }


Mime
View raw message