atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject incubator-atlas git commit: ATLAS-1587: updated AtlasEntityHeader retrieval to read name/description/owner attributes defined in the type (instead of Asset)
Date Wed, 01 Mar 2017 19:08:17 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 3860c95d3 -> 84f6923e1


ATLAS-1587: updated AtlasEntityHeader retrieval to read name/description/owner attributes
defined in the type (instead of Asset)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/84f6923e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/84f6923e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/84f6923e

Branch: refs/heads/master
Commit: 84f6923e12d91307af59c98a9bc6a25fbc9708b6
Parents: 3860c95
Author: Madhan Neethiraj <madhan@apache.org>
Authored: Wed Mar 1 09:11:59 2017 -0800
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Wed Mar 1 10:10:04 2017 -0800

----------------------------------------------------------------------
 .../atlas/discovery/AtlasDiscoveryService.java  |  2 +-
 .../atlas/discovery/EntityDiscoveryService.java | 61 +++++---------------
 .../atlas/discovery/EntityLineageService.java   | 35 +++++------
 .../store/graph/v1/EntityGraphRetriever.java    | 47 ++++++++++++++-
 4 files changed, 73 insertions(+), 72 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/84f6923e/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index e3eb52a..21ce14d 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -39,5 +39,5 @@ public interface AtlasDiscoveryService {
      * @param offset offset to the results returned (for pagination). [ offset >= 0 ].
-1 maps to offset 0.
      * @return AtlasSearchResult
      */
-    AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset);
+    AtlasSearchResult searchUsingFullTextQuery(String query, int limit, int offset) throws
AtlasBaseException;
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/84f6923e/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index da06230..5628ed8 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -17,7 +17,6 @@
  */
 package org.apache.atlas.discovery;
 
-import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
 import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
@@ -25,9 +24,7 @@ import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
 import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.discovery.AtlasSearchResult;
-import org.apache.atlas.model.instance.AtlasEntity.Status;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
-import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
 import org.apache.atlas.query.Expressions.AliasExpression;
 import org.apache.atlas.query.Expressions.Expression;
 import org.apache.atlas.query.Expressions.SelectExpression;
@@ -44,6 +41,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
 import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.slf4j.Logger;
@@ -55,7 +54,6 @@ import scala.util.parsing.combinator.Parsers.NoSuccess;
 import javax.inject.Inject;
 import javax.script.ScriptException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -63,19 +61,17 @@ import java.util.Map;
 import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
 
 public class EntityDiscoveryService implements AtlasDiscoveryService {
-
-    private final AtlasGraph graph;
-    private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
     private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class);
 
-    private final static String PROPERTY_KEY_NAME        = AtlasBaseTypeDef.ATLAS_TYPE_ASSET
+ "." + AtlasClient.NAME;
-    private final static String PROPERTY_KEY_DESCRIPTION = AtlasBaseTypeDef.ATLAS_TYPE_ASSET
+ "." + AtlasClient.DESCRIPTION;
-    private final static String PROPERTY_KEY_OWNER       = AtlasBaseTypeDef.ATLAS_TYPE_ASSET
+ "." + AtlasClient.OWNER;
+    private final AtlasGraph                      graph;
+    private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
+    private final EntityGraphRetriever            entityRetriever;
 
     @Inject
-    EntityDiscoveryService(MetadataRepository metadataRepository) {
-        this.graph = AtlasGraphProvider.getGraphInstance();
+    EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry)
{
+        this.graph                    = AtlasGraphProvider.getGraphInstance();
         this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
+        this.entityRetriever          = new EntityGraphRetriever(typeRegistry);
     }
 
     @Override
@@ -97,7 +93,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
                 if (firstElement instanceof AtlasVertex) {
                     for (Object element : queryResult) {
                         if (element instanceof AtlasVertex) {
-                            ret.addEntity(toAtlasEntityHeader((AtlasVertex)element));
+                            ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex)element));
                         } else {
                             LOG.warn("searchUsingDslQuery({}): expected an AtlasVertex; found
unexpected entry in result {}", dslQuery, element);
                         }
@@ -115,7 +111,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
                                     Object entry = ((List)value).get(0);
 
                                     if (entry instanceof AtlasVertex) {
-                                        ret.addEntity(toAtlasEntityHeader((AtlasVertex)entry));
+                                        ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex)entry));
                                     }
                                 }
                             }
@@ -136,7 +132,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
     }
 
     @Override
-    public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, int limit, int
offset) {
+    public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, int limit, int
offset) throws AtlasBaseException {
         AtlasSearchResult ret      = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT);
         QueryParams       params   = validateSearchParams(limit, offset);
         AtlasIndexQuery   idxQuery = toAtlasIndexQuery(fullTextQuery);
@@ -149,7 +145,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
         return ret;
     }
 
-    private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams
params) {
+    private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams
params) throws AtlasBaseException {
         List<AtlasFullTextResult> ret  = new ArrayList<>();
         Iterator<Result>          iter = query.vertices();
 
@@ -159,7 +155,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
             String guid = vertex != null ? vertex.getProperty(Constants.GUID_PROPERTY_KEY,
String.class) : null;
 
             if (guid != null) {
-                AtlasEntityHeader entity = toAtlasEntityHeader(vertex);
+                AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(vertex);
                 Double score = idxQueryResult.getScore();
                 ret.add(new AtlasFullTextResult(entity, score));
             }
@@ -204,37 +200,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService
{
         return new QueryParams(limit, offset);
     }
 
-    private AtlasEntityHeader toAtlasEntityHeader(AtlasVertex vertex) {
-        if (vertex == null) {
-            return null;
-        }
-
-        AtlasEntityHeader ret = new AtlasEntityHeader();
-
-        String typeName = vertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class);
-
-        ret.setTypeName(typeName);
-        ret.setGuid(vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class));
-        ret.setDisplayText(vertex.getProperty(Constants.QUALIFIED_NAME, String.class));
-        ret.setAttribute(AtlasClient.NAME, vertex.getProperty(PROPERTY_KEY_NAME, String.class));
-        ret.setAttribute(AtlasClient.DESCRIPTION, vertex.getProperty(PROPERTY_KEY_DESCRIPTION,
String.class));
-        ret.setAttribute(AtlasClient.OWNER, vertex.getProperty(PROPERTY_KEY_OWNER, String.class));
-
-        Collection<String> classificationNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY,
String.class);
-
-        if (CollectionUtils.isNotEmpty(classificationNames)) {
-            ret.setClassificationNames(new ArrayList<>(classificationNames));
-        }
-
-        String state = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
-        if (state != null) {
-            Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED);
-            ret.setStatus(status);
-        }
-
-        return ret;
-    }
-
     private AtlasIndexQuery toAtlasIndexQuery(String fullTextQuery) {
         String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY,
fullTextQuery);
         return graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery);

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/84f6923e/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 4a98920..3035d16 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -22,7 +22,6 @@ package org.apache.atlas.discovery;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.instance.AtlasEntity.Status;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.lineage.AtlasLineageInfo;
 import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
@@ -31,6 +30,9 @@ import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graph.AtlasGraphProvider;
 import org.apache.atlas.repository.graphdb.AtlasGraph;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasTypeRegistry;
 import org.apache.atlas.util.AtlasGremlinQueryProvider;
 import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
 import org.apache.commons.collections.CollectionUtils;
@@ -48,13 +50,15 @@ public class EntityLineageService implements AtlasLineageService {
     private static final String INPUT_PROCESS_EDGE      =  "__Process.inputs";
     private static final String OUTPUT_PROCESS_EDGE     =  "__Process.outputs";
 
-    private final AtlasGraph graph;
+    private final AtlasGraph                graph;
     private final AtlasGremlinQueryProvider gremlinQueryProvider;
+    private final EntityGraphRetriever      entityRetriever;
 
     @Inject
-    EntityLineageService() throws DiscoveryException {
-        this.graph = AtlasGraphProvider.getGraphInstance();
+    EntityLineageService(AtlasTypeRegistry typeRegistry) throws DiscoveryException {
+        this.graph                = AtlasGraphProvider.getGraphInstance();
         this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
+        this.entityRetriever      = new EntityGraphRetriever(typeRegistry);
     }
 
     @Override
@@ -99,7 +103,11 @@ public class EntityLineageService implements AtlasLineageService {
                             AtlasEntityHeader prev = null;
 
                             for (Object vertex : vertices) {
-                                AtlasEntityHeader entity = toAtlasEntityHeader(vertex);
+                                if (!(vertex instanceof AtlasVertex)) {
+                                    continue;
+                                }
+
+                                AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader((AtlasVertex)vertex);
 
                                 if (!entities.containsKey(entity.getGuid())) {
                                     entities.put(entity.getGuid(), entity);
@@ -163,23 +171,6 @@ public class EntityLineageService implements AtlasLineageService {
         return lineageQuery;
     }
 
-    private AtlasEntityHeader toAtlasEntityHeader(Object vertexObj) {
-        AtlasEntityHeader ret = new AtlasEntityHeader();
-
-        if (vertexObj instanceof AtlasVertex) {
-            AtlasVertex vertex = (AtlasVertex) vertexObj;
-            ret.setTypeName(vertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class));
-            ret.setGuid(vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class));
-            ret.setDisplayText(vertex.getProperty(Constants.QUALIFIED_NAME, String.class));
-
-            String state  = vertex.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
-            Status status = (state.equalsIgnoreCase("ACTIVE") ? Status.ACTIVE : Status.DELETED);
-            ret.setStatus(status);
-        }
-
-        return ret;
-    }
-
     private boolean entityExists(String guid) {
         boolean ret = false;
         Iterator<AtlasVertex> results = graph.query()

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/84f6923e/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 44ca854..ea5bf93 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -17,8 +17,8 @@
  */
 package org.apache.atlas.repository.store.graph.v1;
 
-import com.google.common.base.Optional;
 import com.sun.istack.Nullable;
+import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.exception.AtlasBaseException;
@@ -27,9 +27,11 @@ import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
 import org.apache.atlas.model.instance.AtlasStruct;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graph.GraphHelper;
 import org.apache.atlas.repository.graphdb.AtlasEdge;
 import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
@@ -120,6 +122,10 @@ public final class EntityGraphRetriever {
         return ret;
     }
 
+    public AtlasEntityHeader toAtlasEntityHeader(AtlasVertex entityVertex) throws AtlasBaseException
{
+        return entityVertex != null ? mapVertexToAtlasEntityHeader(entityVertex) : null;
+    }
+
     private AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
         try {
             return graphHelper.getVertexForGUID(guid);
@@ -171,6 +177,41 @@ public final class EntityGraphRetriever {
         return entity;
     }
 
+    private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) throws
AtlasBaseException {
+        AtlasEntityHeader ret = new AtlasEntityHeader();
+
+        String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class);
+        String guid     = entityVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
+
+        ret.setTypeName(typeName);
+        ret.setGuid(guid);
+        ret.setStatus(GraphHelper.getStatus(entityVertex));
+        ret.setClassificationNames(GraphHelper.getTraitNames(entityVertex));
+
+        AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
+
+        if (entityType != null) {
+            Object name        = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.NAME));
+            Object description = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.DESCRIPTION));
+            Object owner       = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.OWNER));
+            Object displayText = name;
+
+            if (displayText == null) {
+                displayText = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.QUALIFIED_NAME));
+            }
+
+            ret.setAttribute(AtlasClient.NAME, name);
+            ret.setAttribute(AtlasClient.DESCRIPTION, description);
+            ret.setAttribute(AtlasClient.OWNER, owner);
+
+            if (displayText != null) {
+                ret.setDisplayText(displayText.toString());
+            }
+        }
+
+        return ret;
+    }
+
     private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity)
{
         if (LOG.isDebugEnabled()) {
             LOG.debug("Mapping system attributes for type {}", entity.getTypeName());
@@ -495,4 +536,8 @@ public final class EntityGraphRetriever {
 
         return ret;
     }
+
+    private Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) throws
AtlasBaseException {
+        return vertex != null && attribute != null ? mapVertexToAttribute(vertex,
attribute, null) : null;
+    }
 }
\ No newline at end of file


Mime
View raw message