atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject atlas git commit: ATLAS-1988: Implement REST API to search for related entities - Add excludeDeletedEntities flag
Date Thu, 17 Aug 2017 20:51:38 GMT
Repository: atlas
Updated Branches:
  refs/heads/0.8-incubating b8255d418 -> 0183beca0


ATLAS-1988: Implement REST API to search for related entities - Add excludeDeletedEntities
flag


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/0183beca
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/0183beca
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/0183beca

Branch: refs/heads/0.8-incubating
Commit: 0183beca085b95de4e8a24da85bae843011ce732
Parents: b8255d4
Author: Sarath Subramanian <ssubramanian@hortonworks.com>
Authored: Thu Aug 17 13:51:28 2017 -0700
Committer: Sarath Subramanian <ssubramanian@hortonworks.com>
Committed: Thu Aug 17 13:51:28 2017 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/atlas/AtlasClient.java |  1 +
 .../atlas/discovery/AtlasDiscoveryService.java  |  3 +-
 .../atlas/discovery/EntityDiscoveryService.java | 61 ++++++++++++--------
 .../store/graph/v1/EntityGraphRetriever.java    |  2 +
 .../atlas/util/AtlasGremlin2QueryProvider.java  |  8 +--
 .../apache/atlas/web/rest/DiscoveryREST.java    | 17 +++---
 6 files changed, 54 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/client/src/main/java/org/apache/atlas/AtlasClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java
index c50f3f5..5fa5df7 100755
--- a/client/src/main/java/org/apache/atlas/AtlasClient.java
+++ b/client/src/main/java/org/apache/atlas/AtlasClient.java
@@ -101,6 +101,7 @@ public class AtlasClient extends AtlasBaseClient {
     public static final String NAME = "name";
     public static final String DESCRIPTION = "description";
     public static final String OWNER = "owner";
+    public static final String CREATE_TIME = "createTime";
 
     public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure";
     public static final String DATA_SET_SUPER_TYPE = "DataSet";

http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index ead5d3c..8196a67 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -73,9 +73,10 @@ public interface AtlasDiscoveryService {
      * @param relation relation name.
      * @param sortByAttribute sort the result using this attribute name, default value is
'name'
      * @param sortOrder sorting order
+     * @param excludeDeletedEntities exclude deleted entities in search result.
      * @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit
< maxlimit ]. -1 maps to atlas.search.defaultlimit property.
      * @param offset offset to the results returned (for pagination). [ offset >= 0 ].
-1 maps to offset 0.
      * @return AtlasSearchResult
      */
-    AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute,
SortOrder sortOrder, int limit, int offset) throws AtlasBaseException;
+    AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute,
SortOrder sortOrder, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
 }

http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index d5062a7..1e68835 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -76,14 +76,19 @@ import java.util.*;
 import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
 import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
 import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME;
+import static org.apache.atlas.SortOrder.ASCENDING;
 import static org.apache.atlas.SortOrder.DESCENDING;
 import static org.apache.atlas.model.TypeCategory.ARRAY;
 import static org.apache.atlas.model.TypeCategory.MAP;
 import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
+import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
+import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
 import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
+import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER;
 import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH;
 import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_DESCENDING_SORT;
 import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_ASCENDING_SORT;
+import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TO_RANGE_LIST;
 
 @Component
 public class EntityDiscoveryService implements AtlasDiscoveryService {
@@ -364,9 +369,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
             }
 
             if (excludeDeletedEntities) {
-                bindings.put("state", Status.ACTIVE.toString());
+                bindings.put("state", ACTIVE.toString());
 
-                basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER);
+                basicQuery += gremlinQueryProvider.getQuery(BASIC_SEARCH_STATE_FILTER);
             }
 
             if (isGuidPrefixSearch) {
@@ -378,7 +383,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
             bindings.put("startIdx", params.offset());
             bindings.put("endIdx", params.offset() + params.limit());
 
-            basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.TO_RANGE_LIST);
+            basicQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST);
 
             ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
 
@@ -504,8 +509,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
 
     @Override
     @GraphTransaction
-    public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName,
-                                                   SortOrder sortOrder, int limit, int offset)
throws AtlasBaseException {
+    public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName,
SortOrder sortOrder,
+                                                   boolean excludeDeletedEntities, int limit,
int offset) throws AtlasBaseException {
         AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP);
 
         if (StringUtils.isEmpty(guid) || StringUtils.isEmpty(relation)) {
@@ -543,21 +548,37 @@ public class EntityDiscoveryService implements AtlasDiscoveryService
{
             sortByAttributeName = sortByAttribute.getQualifiedName();
 
             if (sortOrder == null) {
-                sortOrder = SortOrder.ASCENDING;
+                sortOrder = ASCENDING;
             }
         }
 
-        String       relatedEntitiesQuery = getRelatedEntitiesQuery(sortOrder);
+        QueryParams  params               = validateSearchParams(limit, offset);
         ScriptEngine scriptEngine         = graph.getGremlinScriptEngine();
         Bindings     bindings             = scriptEngine.createBindings();
-        QueryParams  params               = validateSearchParams(limit, offset);
+        Set<String>  states               = getEntityStates();
+        String       relatedEntitiesQuery = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH);
+
+        if (excludeDeletedEntities) {
+            states.remove(DELETED.toString());
+        }
+
+        if (sortOrder == ASCENDING) {
+            relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT);
+            bindings.put("sortAttributeName", sortByAttributeName);
+
+        } else if (sortOrder == DESCENDING) {
+            relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT);
+            bindings.put("sortAttributeName", sortByAttributeName);
+        }
+
+        relatedEntitiesQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST);
 
         bindings.put("g", graph);
         bindings.put("guid", guid);
         bindings.put("relation", relation);
-        bindings.put("sortAttributeName", sortByAttributeName);
-        bindings.put("offset", params.offset());
-        bindings.put("limit", params.offset() + params.limit());
+        bindings.put("states", Collections.unmodifiableSet(states));
+        bindings.put("startIdx", params.offset());
+        bindings.put("endIdx", params.offset() + params.limit());
 
         try {
             Object result = graph.executeGremlinScript(scriptEngine, bindings, relatedEntitiesQuery,
false);
@@ -725,7 +746,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
     }
 
     private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex<?,
?> vertex) {
-        return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
+        return excludeDeletedEntities && GraphHelper.getStatus(vertex) == DELETED;
     }
 
     private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String
classificationName, int maxTypesLengthInIdxQuery) {
@@ -767,17 +788,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService
{
         return ret;
     }
 
-    private String getRelatedEntitiesQuery(SortOrder sortOrder) {
-        final String ret;
-
-        if (sortOrder == null) {
-            ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH);
-        } else if (sortOrder == DESCENDING) {
-            ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT);
-        } else {
-            ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT);
-        }
-
-        return ret;
+    private Set<String> getEntityStates() {
+        return new HashSet<>(Arrays.asList(ACTIVE.toString(), DELETED.toString()));
     }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index f6c8c72..83c9c2b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -218,11 +218,13 @@ public final class EntityGraphRetriever {
             Object name        = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.NAME));
             Object description = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.DESCRIPTION));
             Object owner       = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.OWNER));
+            Object createTime  = entityVertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY,
Long.class);
             Object displayText = name != null ? name : ret.getAttribute(AtlasClient.QUALIFIED_NAME);
 
             ret.setAttribute(AtlasClient.NAME, name);
             ret.setAttribute(AtlasClient.DESCRIPTION, description);
             ret.setAttribute(AtlasClient.OWNER, owner);
+            ret.setAttribute(AtlasClient.CREATE_TIME, createTime);
 
             if (displayText != null) {
                 ret.setDisplayText(displayText.toString());

http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
index 2869e88..ea4676a 100644
--- a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
+++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
@@ -76,11 +76,11 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider
{
             case GUID_PREFIX_FILTER:
                 return ".filter{it.'__guid'.matches(guid)}";
             case RELATIONSHIP_SEARCH:
-                return "g.V('__guid', guid).both(relation)[offset..<limit].toList()";
-            case RELATIONSHIP_SEARCH_DESCENDING_SORT:
-                return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.b.getProperty(sortAttributeName)
<=> it.a.getProperty(sortAttributeName)}.toList()";
+                return "g.V('__guid', guid).both(relation).has('__state', T.in, states)";
             case RELATIONSHIP_SEARCH_ASCENDING_SORT:
-                return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.a.getProperty(sortAttributeName)
<=> it.b.getProperty(sortAttributeName)}.toList()";
+                return ".order{it.a.getProperty(sortAttributeName) <=> it.b.getProperty(sortAttributeName)}";
+            case RELATIONSHIP_SEARCH_DESCENDING_SORT:
+                return ".order{it.b.getProperty(sortAttributeName) <=> it.a.getProperty(sortAttributeName)}";
             case COMPARE_LT:
                 return ".has('%s', T.lt, %s)";
             case COMPARE_LTE:

http://git-wip-us.apache.org/repos/asf/atlas/blob/0183beca/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index 032bde8..52258e3 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -285,21 +285,22 @@ public class DiscoveryREST {
     @Path("relationship")
     @Consumes(Servlets.JSON_MEDIA_TYPE)
     @Produces(Servlets.JSON_MEDIA_TYPE)
-    public AtlasSearchResult searchRelatedEntities(@QueryParam("guid")      String    guid,
-                                                   @QueryParam("relation")  String    relation,
-                                                   @QueryParam("sortBy")    String    sortByAttribute,
-                                                   @QueryParam("sortOrder") SortOrder sortOrder,
-                                                   @QueryParam("limit")     int       limit,
-                                                   @QueryParam("offset")    int       offset)
throws AtlasBaseException {
+    public AtlasSearchResult searchRelatedEntities(@QueryParam("guid")      String      
guid,
+                                                   @QueryParam("relation")  String      
relation,
+                                                   @QueryParam("sortBy")    String      
sortByAttribute,
+                                                   @QueryParam("sortOrder") SortOrder   
sortOrder,
+                                                   @QueryParam("excludeDeletedEntities")
boolean excludeDeletedEntities,
+                                                   @QueryParam("limit")     int         
limit,
+                                                   @QueryParam("offset")    int         
offset) throws AtlasBaseException {
         AtlasPerfTracer perf = null;
 
         try {
             if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
                 perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.relatedEntitiesSearchUsingGremlin("
+ guid +
-                        ", " + relation + ", " + sortByAttribute + ", " + sortOrder + ",
" + limit + ", " + offset + ")");
+                        ", " + relation + ", " + sortByAttribute + ", " + sortOrder + ",
" + excludeDeletedEntities + ", " + ", " + limit + ", " + offset + ")");
             }
 
-            return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute,
sortOrder, limit, offset);
+            return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute,
sortOrder, excludeDeletedEntities, limit, offset);
         } finally {
             AtlasPerfTracer.log(perf);
         }


Mime
View raw message