atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject [4/5] incubator-atlas git commit: ATLAS-622 Introduce soft delete (shwethags)
Date Sat, 16 Apr 2016 03:58:13 GMT
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index b94ff5a..de29e86 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -19,14 +19,14 @@
 package org.apache.atlas.repository.graph;
 
 import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
 import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
-
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.GraphTransaction;
+import org.apache.atlas.RequestContext;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.RepositoryException;
@@ -37,15 +37,13 @@ import org.apache.atlas.typesystem.exception.EntityNotFoundException;
 import org.apache.atlas.typesystem.exception.TraitNotFoundException;
 import org.apache.atlas.typesystem.types.AttributeInfo;
 import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
 import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.typesystem.types.TypeUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -60,18 +58,21 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
 
     private static final Logger LOG = LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
 
-    private final GraphToTypedInstanceMapper graphToInstanceMapper;
-
     private static TypeSystem typeSystem = TypeSystem.getInstance();
 
     private static final GraphHelper graphHelper = GraphHelper.getInstance();
 
     private final TitanGraph titanGraph;
 
+    private DeleteHandler deleteHandler;
+
+    private GraphToTypedInstanceMapper graphToInstanceMapper;
+
     @Inject
-    public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) {
+    public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider, DeleteHandler
deleteHandler) {
         this.titanGraph = graphProvider.get();
-        this.graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph);
+        graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph);
+        this.deleteHandler = deleteHandler;
     }
 
     public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
@@ -122,10 +123,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
         EntityExistsException {
         LOG.info("adding entities={}", entities);
         try {
-            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
-            TypeUtils.Pair<List<String>, List<String>> idPair =
-                    instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE,
entities);
-            return idPair.left;
+            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper,
deleteHandler);
+            instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE,
entities);
+            return RequestContext.get().getCreatedEntityIds();
         } catch (EntityExistsException e) {
             throw e;
         } catch (AtlasException e) {
@@ -215,14 +215,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
             // add the trait instance as a new vertex
             final String typeName = GraphHelper.getTypeName(instanceVertex);
 
-            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
+            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper,
deleteHandler);
             instanceToGraphMapper.mapTraitInstanceToVertex(traitInstance,
                     typeSystem.getDataType(ClassType.class, typeName), instanceVertex);
 
 
             // update the traits in entity once adding trait instance is successful
             GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
-            GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
Long.valueOf(System.currentTimeMillis()));
+            GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
+                    RequestContext.get().getRequestTime());
             
         } catch (RepositoryException e) {
             throw e;
@@ -254,23 +255,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
         try {
             final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
             String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted);
-            Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
-            if (results.hasNext()) { // there should only be one edge for this label
-                final Edge traitEdge = results.next();
-                final Vertex traitVertex = traitEdge.getVertex(Direction.IN);
-
-                // remove the edge to the trait instance from the repository
-                titanGraph.removeEdge(traitEdge);
-
-                if (traitVertex != null) { // remove the trait instance from the repository
-                    TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
-                    instanceToGraphMapper.deleteTraitVertex(traitNameToBeDeleted, traitVertex);
-
-                    // update the traits in entity once trait removal is successful
-                    traitNames.remove(traitNameToBeDeleted);
-                    updateTraits(instanceVertex, traitNames);
-                }
-            }
+
+            deleteHandler.deleteReference(instanceVertex, relationshipLabel, DataTypes.TypeCategory.TRAIT);
+
+            // update the traits in entity once trait removal is successful
+            traitNames.remove(traitNameToBeDeleted);
+            updateTraits(instanceVertex, traitNames);
         } catch (Exception e) {
             throw new RepositoryException(e);
         }
@@ -285,7 +275,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
         for (String traitName : traitNames) {
             GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
         }
-        GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
Long.valueOf(System.currentTimeMillis()));
+        GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
+                RequestContext.get().getRequestTime());
     }
 
     @Override
@@ -293,9 +284,11 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
     public TypeUtils.Pair<List<String>, List<String>> updateEntities(ITypedReferenceableInstance...
entitiesUpdated) throws RepositoryException {
         LOG.info("updating entity {}", entitiesUpdated);
         try {
-            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
-            return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
+            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper,
deleteHandler);
+            instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
                     entitiesUpdated);
+            RequestContext requestContext = RequestContext.get();
+            return TypeUtils.Pair.of(requestContext.getCreatedEntityIds(), requestContext.getUpdatedEntityIds());
         } catch (AtlasException e) {
             throw new RepositoryException(e);
         }
@@ -306,8 +299,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
     public TypeUtils.Pair<List<String>, List<String>> updatePartial(ITypedReferenceableInstance
entity) throws RepositoryException {
         LOG.info("updating entity {}", entity);
         try {
-            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
-            return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL,
entity);
+            TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper,
deleteHandler);
+            instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL,
entity);
+            RequestContext requestContext = RequestContext.get();
+            return TypeUtils.Pair.of(requestContext.getCreatedEntityIds(), requestContext.getUpdatedEntityIds());
         } catch (AtlasException e) {
             throw new RepositoryException(e);
         }
@@ -315,13 +310,12 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
 
     @Override
     @GraphTransaction
-    public TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>>
 deleteEntities(List<String> guids) throws RepositoryException {
+    public TypeUtils.Pair<List<String>, List<ITypedReferenceableInstance>>
deleteEntities(List<String> guids) throws RepositoryException {
 
         if (guids == null || guids.size() == 0) {
             throw new IllegalArgumentException("guids must be non-null and non-empty");
         }
         
-        TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
         for (String guid : guids) {
             if (guid == null) {
                 LOG.warn("deleteEntities: Ignoring null guid");
@@ -329,8 +323,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
             }
             try {
                 Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
-                String typeName = GraphHelper.getTypeName(instanceVertex);
-                instanceToGraphMapper.deleteEntity(typeName, instanceVertex);
+                deleteHandler.deleteEntity(instanceVertex);
             } catch (EntityNotFoundException e) {
                 // Entity does not exist - treat as non-error, since the caller
                 // wanted to delete the entity and it's already gone.
@@ -340,7 +333,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository
{
                 throw new RepositoryException(e);
             }
         }
-        return new TypeUtils.Pair<>(
-                instanceToGraphMapper.getDeletedEntityGuids(), instanceToGraphMapper.getDeletedEntities());
+        RequestContext requestContext = RequestContext.get();
+        return new TypeUtils.Pair<>(requestContext.getDeletedEntityIds(), requestContext.getDeletedEntities());
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index e7e8fb9..d83c08c 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -100,7 +100,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
             management.buildIndex(Constants.EDGE_INDEX, Edge.class).buildMixedIndex(Constants.BACKING_INDEX);
 
             // create a composite index for guid as its unique
-            createCompositeAndMixedIndex(management, Constants.GUID_PROPERTY_KEY, String.class,
true, Cardinality.SINGLE, true);
+            createCompositeAndMixedIndex(management, Constants.GUID_PROPERTY_KEY, String.class,
true,
+                    Cardinality.SINGLE, true);
+
+            // create a composite index for entity state
+            createCompositeAndMixedIndex(management, Constants.STATE_PROPERTY_KEY, String.class,
false, Cardinality.SINGLE, true);
 
             // create a composite and mixed index for type since it can be combined with
other keys
             createCompositeAndMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY,
String.class, false, Cardinality.SINGLE,
@@ -223,13 +227,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
         switch (field.dataType().getTypeCategory()) {
         case PRIMITIVE:
             Cardinality cardinality = getCardinality(field.multiplicity);
-            createCompositeAndMixedIndex(management, propertyName, getPrimitiveClass(field.dataType()),
field.isUnique,
+            createCompositeAndMixedIndex(management, propertyName, getPrimitiveClass(field.dataType()),
false,
                     cardinality, false);
             break;
 
         case ENUM:
             cardinality = getCardinality(field.multiplicity);
-            createCompositeAndMixedIndex(management, propertyName, String.class, field.isUnique,
cardinality, false);
+            createCompositeAndMixedIndex(management, propertyName, String.class, false, cardinality,
false);
             break;
 
         case ARRAY:

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 895f9df..c542ec7 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -23,11 +23,12 @@ import com.thinkaurelius.titan.core.TitanProperty;
 import com.thinkaurelius.titan.core.TitanVertex;
 import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Element;
 import com.tinkerpop.blueprints.Graph;
 import com.tinkerpop.blueprints.GraphQuery;
 import com.tinkerpop.blueprints.Vertex;
-
 import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.typesystem.IReferenceableInstance;
 import org.apache.atlas.typesystem.ITypedInstance;
@@ -40,7 +41,6 @@ import org.apache.atlas.typesystem.types.DataTypes;
 import org.apache.atlas.typesystem.types.HierarchicalType;
 import org.apache.atlas.typesystem.types.IDataType;
 import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUtils.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -77,7 +77,7 @@ public final class GraphHelper {
         final String guid = UUID.randomUUID().toString();
 
         final Vertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(),
-                new Id(guid, 0 , typedInstance.getTypeName()), superTypeNames);
+                new Id(guid, 0, typedInstance.getTypeName()), superTypeNames);
 
         // add identity
         setProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid);
@@ -85,9 +85,6 @@ public final class GraphHelper {
         // add version information
         setProperty(vertexWithIdentity, Constants.VERSION_PROPERTY_KEY, typedInstance.getId().version);
 
-        // add state information
-        setProperty(vertexWithIdentity, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
-
         return vertexWithIdentity;
     }
 
@@ -99,41 +96,120 @@ public final class GraphHelper {
         // add type information
         setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
 
+
         // add super types
         for (String superTypeName : superTypeNames) {
             addProperty(vertexWithoutIdentity, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
         }
 
+        // add state information
+        setProperty(vertexWithoutIdentity, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
+
         // add timestamp information
-        setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
+        setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
 
         return vertexWithoutIdentity;
     }
 
     public Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
-        LOG.debug("Adding edge for {} -> label {} -> {}", fromVertex, edgeLabel, toVertex);
+        LOG.debug("Adding edge for {} -> label {} -> {}", string(fromVertex), edgeLabel,
string(toVertex));
         Edge edge = titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
-        LOG.debug("Added edge for {} -> label {}, id {} -> {}", fromVertex, edgeLabel,
edge.getId(), toVertex);
+
+        setProperty(edge, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
+        setProperty(edge, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
+        setProperty(edge, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
+
+        LOG.debug("Added {}", string(edge));
         return edge;
     }
 
-    public Vertex findVertex(String propertyKey, Object value) {
-        LOG.debug("Finding vertex for {}={}", propertyKey, value);
+    public Edge getOrCreateEdge(Vertex outVertex, Vertex inVertex, String edgeLabel) {
+        Iterable<Edge> edges = inVertex.getEdges(Direction.IN, edgeLabel);
+        for (Edge edge : edges) {
+            if (edge.getVertex(Direction.OUT).getId().toString().equals(outVertex.getId().toString()))
{
+                return edge;
+            }
+        }
+        return addEdge(outVertex, inVertex, edgeLabel);
+    }
+
+    /**
+     * Args of the format prop1, key1, prop2, key2...
+     * Searches for a vertex with prop1=key1 && prop2=key2
+     * @param args
+     * @return vertex with the given property keys
+     * @throws EntityNotFoundException
+     */
+    private Vertex findVertex(Object... args) throws EntityNotFoundException {
+        StringBuilder condition = new StringBuilder();
+        GraphQuery query = titanGraph.query();
+        for (int i = 0 ; i < args.length; i+=2) {
+            query = query.has((String) args[i], args[i+1]);
+            condition.append(args[i]).append(" = ").append(args[i+1]).append(", ");
+        }
+        String conditionStr = condition.toString();
+        LOG.debug("Finding vertex with {}", conditionStr);
 
-        GraphQuery query = titanGraph.query().has(propertyKey, value);
         Iterator<Vertex> results = query.vertices().iterator();
         // returning one since entityType, qualifiedName should be unique
-        return results.hasNext() ? results.next() : null;
+        Vertex vertex = results.hasNext() ? results.next() : null;
+
+        if (vertex == null) {
+            LOG.debug("Could not find a vertex with {}", condition.toString());
+            throw new EntityNotFoundException("Could not find an entity in the repository
with " + conditionStr);
+        } else {
+            LOG.debug("Found a vertex {} with {}", string(vertex), conditionStr);
+        }
+
+        return vertex;
     }
 
-    public static Iterable<Edge> getOutGoingEdgesByLabel(Vertex instanceVertex, String
edgeLabel) {
+    public static Iterator<Edge> getOutGoingEdgesByLabel(Vertex instanceVertex, String
edgeLabel) {
+        LOG.debug("Finding edges for {} with label {}", string(instanceVertex), edgeLabel);
         if(instanceVertex != null && edgeLabel != null) {
-            return instanceVertex.getEdges(Direction.OUT, edgeLabel);
+            return instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator();
         }
         return null;
     }
 
-    public Edge getOutGoingEdgeById(String edgeId) {
+    /**
+     * Returns the active edge for the given edge label.
+     * If the vertex is deleted and there is no active edge, it returns the latest deleted
edge
+     * @param vertex
+     * @param edgeLabel
+     * @return
+     */
+    public static Edge getEdgeForLabel(Vertex vertex, String edgeLabel) {
+        String vertexState = vertex.getProperty(Constants.STATE_PROPERTY_KEY);
+
+        Iterator<Edge> iterator = GraphHelper.getOutGoingEdgesByLabel(vertex, edgeLabel);
+        Edge latestDeletedEdge = null;
+        long latestDeletedEdgeTime = Long.MIN_VALUE;
+        while (iterator != null && iterator.hasNext()) {
+            Edge edge = iterator.next();
+            String edgeState = edge.getProperty(Constants.STATE_PROPERTY_KEY);
+            if (edgeState == null || Id.EntityState.ACTIVE.name().equals(edgeState)) {
+                LOG.debug("Found {}", string(edge));
+                return edge;
+            } else {
+                Long modificationTime = edge.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+                if (modificationTime != null && modificationTime >= latestDeletedEdgeTime)
{
+                    latestDeletedEdgeTime = modificationTime;
+                    latestDeletedEdge = edge;
+                }
+            }
+        }
+
+        //If the vertex is deleted, return latest deleted edge
+        if (Id.EntityState.DELETED.equals(vertexState)) {
+            LOG.debug("Found {}", string(latestDeletedEdge));
+            return latestDeletedEdge;
+        }
+
+        return null;
+    }
+
+    public Edge getEdgeById(String edgeId) {
         if(edgeId != null) {
             return titanGraph.getEdge(edgeId);
         }
@@ -154,60 +230,47 @@ public final class GraphHelper {
                 + edge.getVertex(Direction.IN) + "]";
     }
 
-    public static void setProperty(Vertex vertex, String propertyName, Object value) {
-        LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
-        Object existValue = vertex.getProperty(propertyName);
+    public static <T extends Element> void setProperty(T element, String propertyName,
Object value) {
+        String elementStr = string(element);
+        LOG.debug("Setting property {} = \"{}\" to {}", propertyName, value, elementStr);
+        Object existValue = element.getProperty(propertyName);
         if(value == null || (value instanceof Collection && ((Collection) value).isEmpty()))
{
             if(existValue != null) {
-                LOG.info("Removing property - {} value from vertex {}", propertyName, vertex);
-                vertex.removeProperty(propertyName);
+                LOG.info("Removing property - {} value from {}", propertyName, elementStr);
+                element.removeProperty(propertyName);
             }
         } else {
             if (!value.equals(existValue)) {
-                vertex.setProperty(propertyName, value);
-                LOG.debug("Set property {} = \"{}\" to vertex {}", propertyName, value, vertex);
+                element.setProperty(propertyName, value);
+                LOG.debug("Set property {} = \"{}\" to {}", propertyName, value, elementStr);
             }
         }
     }
 
+    private static <T extends Element> String string(T element) {
+        if (element instanceof Vertex) {
+            return string((Vertex) element);
+        } else if (element instanceof Edge) {
+            return string((Edge)element);
+        }
+        return element.toString();
+    }
+
     public static void addProperty(Vertex vertex, String propertyName, Object value) {
-        LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
+        LOG.debug("Adding property {} = \"{}\" to vertex {}", propertyName, value, string(vertex));
         ((TitanVertex)vertex).addProperty(propertyName, value);
     }
 
-    public Edge removeRelation(String edgeId, boolean cascade) {
-        LOG.debug("Removing edge with id {}", edgeId);
-        final Edge edge = titanGraph.getEdge(edgeId);
-        titanGraph.removeEdge(edge);
-        LOG.info("Removed edge {}", edge);
-        if (cascade) {
-           Vertex referredVertex = edge.getVertex(Direction.IN);
-           removeVertex(referredVertex);
-        }
-        return edge;
-    }
-    
     /**
      * Remove the specified edge from the graph.
      * 
      * @param edge
      */
     public void removeEdge(Edge edge) {
-        LOG.debug("Removing edge {}", edge);
+        String edgeString = string(edge);
+        LOG.debug("Removing {}", edgeString);
         titanGraph.removeEdge(edge);
-        LOG.info("Removed edge {}", edge);
-    }
-    
-    /**
-     * Return the edge and target vertex for the specified edge ID.
-     * 
-     * @param edgeId
-     * @return edge and target vertex
-     */
-    public Pair<Edge, Vertex> getEdgeAndTargetVertex(String edgeId) {
-        final Edge edge = titanGraph.getEdge(edgeId);
-        Vertex referredVertex = edge.getVertex(Direction.IN);
-        return Pair.of(edge, referredVertex);
+        LOG.info("Removed {}", edgeString);
     }
     
     /**
@@ -216,27 +279,22 @@ public final class GraphHelper {
      * @param vertex
      */
     public void removeVertex(Vertex vertex) {
-        LOG.debug("Removing vertex {}", vertex);
+        String vertexString = string(vertex);
+        LOG.debug("Removing {}", vertexString);
         titanGraph.removeVertex(vertex);
-        LOG.info("Removed vertex {}", vertex);
+        LOG.info("Removed {}", vertexString);
     }
 
     public Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
-        return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid);
+        return findVertex(Constants.GUID_PROPERTY_KEY, guid);
     }
 
-
     public Vertex getVertexForProperty(String propertyKey, Object value) throws EntityNotFoundException
{
-        Vertex instanceVertex = findVertex(propertyKey, value);
-        if (instanceVertex == null) {
-            LOG.debug("Could not find a vertex with {}={}", propertyKey, value);
-            throw new EntityNotFoundException("Could not find an entity in the repository
with " + propertyKey + "="
-                + value);
-        } else {
-            LOG.debug("Found a vertex {} with {}={}", instanceVertex, propertyKey, value);
-        }
+        return findVertex(propertyKey, value, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
+    }
 
-        return instanceVertex;
+    public static String getQualifiedNameForMapKey(String prefix, String key) {
+        return prefix + "." + key;
     }
 
     public static String getQualifiedFieldName(ITypedInstance typedInstance, AttributeInfo
attributeInfo) throws AtlasException {
@@ -277,6 +335,10 @@ public final class GraphHelper {
             vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName);
     }
 
+    public static String getIdFromVertex(Vertex vertex) {
+        return vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY);
+    }
+
     public static String getTypeName(Vertex instanceVertex) {
         return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
     }
@@ -292,7 +354,7 @@ public final class GraphHelper {
      */
     public Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance
instance)
         throws AtlasException {
-        LOG.debug("Checking if there is an instance with the same unique attributes for instance
{}", instance);
+        LOG.debug("Checking if there is an instance with the same unique attributes for instance
{}", instance.toShortString());
         Vertex result = null;
         for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
             if (attributeInfo.isUnique) {
@@ -322,4 +384,18 @@ public final class GraphHelper {
         }
         LOG.debug("*******************Graph Dump****************************");
     }
+
+    public static String string(ITypedReferenceableInstance instance) {
+        return String.format("entity[type=%s guid=%]", instance.getTypeName(), instance.getId()._getId());
+    }
+
+    public static String string(Vertex vertex) {
+        return String.format("vertex[id=%s type=%s guid=%s]", vertex.getId().toString(),
getTypeName(vertex),
+                getIdFromVertex(vertex));
+    }
+
+    public static String string(Edge edge) {
+        return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getId().toString(),
edge.getLabel(),
+                string(edge.getVertex(Direction.OUT)), string(edge.getVertex(Direction.IN)));
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
index 1d682bb..df28ab3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
@@ -17,6 +17,7 @@
  */
 package org.apache.atlas.repository.graph;
 
+import com.google.inject.Singleton;
 import com.thinkaurelius.titan.core.TitanGraph;
 import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
@@ -43,15 +44,19 @@ import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.atlas.repository.graph.GraphHelper.string;
+
+@Singleton
 public final class GraphToTypedInstanceMapper {
 
     private static final Logger LOG = LoggerFactory.getLogger(GraphToTypedInstanceMapper.class);
     private static TypeSystem typeSystem = TypeSystem.getInstance();
-    private final TitanGraph titanGraph;
+    private static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+    private TitanGraph titanGraph;
 
     public GraphToTypedInstanceMapper(TitanGraph titanGraph) {
         this.titanGraph = titanGraph;
@@ -97,12 +102,12 @@ public final class GraphToTypedInstanceMapper {
         }
     }
 
-
     private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
         AttributeInfo attributeInfo) throws AtlasException {
         LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
         final IDataType dataType = attributeInfo.dataType();
         final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance,
attributeInfo);
+        String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
 
         switch (dataType.getTypeCategory()) {
         case PRIMITIVE:
@@ -128,7 +133,9 @@ public final class GraphToTypedInstanceMapper {
             break;
 
         case STRUCT:
-            mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
+            ITypedStruct structInstance = mapVertexToStructInstance(instanceVertex,
+                    (StructType) attributeInfo.dataType(), relationshipLabel, null);
+            typedInstance.set(attributeInfo.name, structInstance);
             break;
 
         case TRAIT:
@@ -136,9 +143,8 @@ public final class GraphToTypedInstanceMapper {
             break;
 
         case CLASS:
-            String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
             Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo,
relationshipLabel,
-                attributeInfo.dataType());
+                attributeInfo.dataType(), null);
             if (idOrInstance != null) {
                 typedInstance.set(attributeInfo.name, idOrInstance);
             }
@@ -150,25 +156,30 @@ public final class GraphToTypedInstanceMapper {
     }
 
     private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
-        String relationshipLabel, IDataType dataType) throws AtlasException {
+        String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException
{
         LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
-        Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
-        if (results.hasNext()) {
-            final Vertex referenceVertex = results.next().getVertex(Direction.IN);
-            if (referenceVertex != null) {
-                final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
-                LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel,
guid);
-                if (attributeInfo.isComposite) {
-                    //Also, when you retrieve a type's instance, you get the complete object
graph of the composites
-                    LOG.debug("Found composite, mapping vertex to instance");
-                    return mapGraphToTypedInstance(guid, referenceVertex);
-                } else {
-                    Id referenceId =
+
+        Edge edge;
+        if (edgeId == null) {
+            edge = GraphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);;
+        } else {
+            edge = graphHelper.getEdgeById(edgeId);
+        }
+
+        if (edge != null) {
+            final Vertex referenceVertex = edge.getVertex(Direction.IN);
+            final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+            LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel,
guid);
+            if (attributeInfo.isComposite) {
+                //Also, when you retrieve a type's instance, you get the complete object
graph of the composites
+                LOG.debug("Found composite, mapping vertex to instance");
+                return mapGraphToTypedInstance(guid, referenceVertex);
+            } else {
+                Id referenceId =
                         new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
-                            dataType.getName());
-                    LOG.debug("Found non-composite, adding id {} ", referenceId);
-                    return referenceId;
-                }
+                                dataType.getName());
+                LOG.debug("Found non-composite, adding id {} ", referenceId);
+                return referenceId;
             }
         }
 
@@ -212,8 +223,7 @@ public final class GraphToTypedInstanceMapper {
             break;
 
         case STRUCT:
-            return getStructInstanceFromVertex(instanceVertex, elementType, attributeInfo.name,
edgeLabel,
-                (String) value);
+            return mapVertexToStructInstance(instanceVertex, (StructType) elementType, edgeLabel,
(String) value);
 
         case CLASS:
             return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType,
(String) value);
@@ -252,83 +262,27 @@ public final class GraphToTypedInstanceMapper {
         }
     }
 
-    private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, IDataType elemType,
-        String attributeName, String relationshipLabel, String edgeId) throws AtlasException
{
-        LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
-        for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
-            if (edgeId.equals(String.valueOf(edge.getId()))) {
-                Vertex structInstanceVertex = edge.getVertex(Direction.IN);
-                LOG.debug("mapping vertex {} to struct {}", structInstanceVertex, attributeName);
-
-                if (structInstanceVertex != null) {
-                    LOG.debug("Found struct instance vertex {}, mapping to instance {} ",
structInstanceVertex,
-                        elemType.getName());
-                    StructType structType = typeSystem.getDataType(StructType.class, elemType.getName());
-                    ITypedStruct structInstance = structType.createInstance();
-                    mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
-                    return structInstance;
-                }
-
-                break;
-            }
-        }
-
-        return null;
-    }
-
-    private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
-        String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException
{
-        LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
-        for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
-            if (edgeId.equals(String.valueOf(edge.getId()))) {
-                final Vertex referenceVertex = edge.getVertex(Direction.IN);
-                if (referenceVertex != null) {
-                    final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
-                    LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex,
relationshipLabel,
-                        guid);
-                    if (attributeInfo.isComposite) {
-                        //Also, when you retrieve a type's instance, you get the complete
object graph of the composites
-                        LOG.debug("Found composite, mapping vertex to instance");
-                        return mapGraphToTypedInstance(guid, referenceVertex);
-                    } else {
-                        Id referenceId =
-                            new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
-                                dataType.getName());
-                        LOG.debug("Found non-composite, adding id {} ", referenceId);
-                        return referenceId;
-                    }
-                }
-
-                break;
-            }
-        }
-
-        return null;
-    }
-
-    private void mapVertexToStructInstance(Vertex instanceVertex, ITypedInstance typedInstance,
-        AttributeInfo attributeInfo) throws AtlasException {
-        LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
-        StructType structType = typeSystem.getDataType(StructType.class, attributeInfo.dataType().getName());
+    private ITypedStruct mapVertexToStructInstance(Vertex instanceVertex, StructType structType,
+                                                   String relationshipLabel, String edgeId)
throws AtlasException {
+        LOG.debug("mapping {} to struct {}", string(instanceVertex), relationshipLabel);
         ITypedStruct structInstance = null;
 
-        String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
-        LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
-        final Iterable<Edge> edges = instanceVertex.getEdges(Direction.OUT, relationshipLabel);
-        if (edges.iterator().hasNext()) {
-            structInstance = structType.createInstance();
-            typedInstance.set(attributeInfo.name, structInstance);
+        Edge edge;
+        if (edgeId == null) {
+            edge = GraphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
+        } else {
+            edge = graphHelper.getEdgeById(edgeId);
         }
 
-        for (Edge edge : edges) {
-            final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
-            if (structInstanceVertex != null) {
-                LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
+        if (edge != null) {
+            structInstance = structType.createInstance();
+            Vertex structInstanceVertex = edge.getVertex(Direction.IN);
+            LOG.debug("Found struct instance {}, mapping to instance {} ", string(structInstanceVertex),
                     structInstance.getTypeName());
-                mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
-                break;
-            }
+            mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
+
         }
+        return structInstance;
     }
 
     private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance
typedInstance,

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java
b/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java
new file mode 100644
index 0000000..f8bbf73
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.inject.Inject;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.TypeSystem;
+
+public class HardDeleteHandler extends DeleteHandler {
+
+    private static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+    @Inject
+    public HardDeleteHandler(TypeSystem typeSystem) {
+        super(typeSystem, true);
+    }
+
+    @Override
+    protected void _deleteVertex(Vertex instanceVertex) {
+        graphHelper.removeVertex(instanceVertex);
+    }
+
+    @Override
+    protected void deleteEdge(Edge edge) throws AtlasException {
+        graphHelper.removeEdge(edge);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/40ee9492/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java
b/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java
new file mode 100644
index 0000000..aa78582
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.inject.Inject;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.TypeSystem;
+
+import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+
+public class SoftDeleteHandler extends DeleteHandler {
+    @Inject
+    public SoftDeleteHandler(TypeSystem typeSystem) {
+        super(typeSystem, false);
+    }
+
+    @Override
+    protected void _deleteVertex(Vertex instanceVertex) {
+        Id.EntityState state = Id.EntityState.valueOf((String) instanceVertex.getProperty(STATE_PROPERTY_KEY));
+        if (state != Id.EntityState.DELETED) {
+            GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
+            GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
+        }
+    }
+
+    @Override
+    protected void deleteEdge(Edge edge) throws AtlasException {
+        Id.EntityState state = Id.EntityState.valueOf((String) edge.getProperty(STATE_PROPERTY_KEY));
+        if (state != Id.EntityState.DELETED) {
+            GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
+            GraphHelper.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
+        }
+    }
+}



Mime
View raw message