atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject incubator-atlas git commit: ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering
Date Tue, 18 Jul 2017 15:36:11 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/master 71f9c0652 -> 377fe19f0


ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/377fe19f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/377fe19f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/377fe19f

Branch: refs/heads/master
Commit: 377fe19f023a4036f9fa1503dec6ea698fdb214b
Parents: 71f9c06
Author: Madhan Neethiraj <madhan@apache.org>
Authored: Tue Jul 18 00:25:36 2017 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Tue Jul 18 08:05:53 2017 -0700

----------------------------------------------------------------------
 .../atlas/discovery/EntitySearchProcessor.java  | 73 ++++++++++----------
 .../apache/atlas/discovery/SearchContext.java   |  2 +-
 .../apache/atlas/discovery/SearchProcessor.java | 14 +++-
 3 files changed, 50 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 605cb15..3204ecf 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
 import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.commons.collections.CollectionUtils;
@@ -34,60 +33,70 @@ public class EntitySearchProcessor extends SearchProcessor {
     private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor");
 
     private final AtlasIndexQuery indexQuery;
-    private final AtlasGraphQuery partialGraphQuery;
-    private final AtlasGraphQuery allGraphQuery;
+    private final AtlasGraphQuery graphQuery;
+    private final AtlasGraphQuery filterGraphQuery;
 
     public EntitySearchProcessor(SearchContext context) {
         super(context);
 
-        AtlasEntityType         entityType         = context.getEntityType();
-        AtlasClassificationType classificationType = context.getClassificationType();
-        FilterCriteria          filterCriteria     = context.getSearchParameters().getEntityFilters();
-        Set<String>             typeAndSubTypes    = entityType.getTypeAndAllSubTypes();
-        Set<String>             solrAttributes     = new HashSet<>();
-        Set<String>             gremlinAttributes  = new HashSet<>();
-        Set<String>             allAttributes      = new HashSet<>();
+        final AtlasEntityType entityType         = context.getEntityType();
+        final FilterCriteria  filterCriteria     = context.getSearchParameters().getEntityFilters();
+        final Set<String>     typeAndSubTypes    = entityType.getTypeAndAllSubTypes();
+        final Set<String>     solrAttributes     = new HashSet<>();
+        final Set<String>     gremlinAttributes  = new HashSet<>();
+        final Set<String>     allAttributes      = new HashSet<>();
 
 
         processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes,
allAttributes);
 
-        boolean useSolrSearch = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY
&& canApplySolrFilter(entityType, filterCriteria, false);
+        final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+        final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false);
 
-        if (useSolrSearch) {
-            StringBuilder solrQuery = new StringBuilder();
+        StringBuilder solrQuery = new StringBuilder();
 
+        if (typeSearchBySolr) {
             constructTypeTestQuery(solrQuery, typeAndSubTypes);
+        }
+
+        if (attrSearchBySolr) {
             constructFilterQuery(solrQuery, entityType, filterCriteria, solrAttributes);
+        } else {
+            gremlinAttributes.addAll(solrAttributes);
+        }
 
+        if (solrQuery.length() > 0) {
             String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
 
             solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
             solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
 
             indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
+        } else {
+            indexQuery = null;
+        }
 
-            if (CollectionUtils.isNotEmpty(gremlinAttributes) || classificationType != null)
{
-                AtlasGraphQuery query = context.getGraph().query();
+        if (CollectionUtils.isNotEmpty(gremlinAttributes) || !typeSearchBySolr) {
+            AtlasGraphQuery query = context.getGraph().query();
 
-                addClassificationNameConditionIfNecessary(query);
+            if (!typeSearchBySolr) {
+                query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+            }
+
+            graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes,
query);
 
-                partialGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes,
query);
-            } else {
-                partialGraphQuery = null;
+            if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery
== null) {
+                graphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
             }
         } else {
-            indexQuery      = null;
-            partialGraphQuery = null;
+            graphQuery = null;
         }
 
         AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY,
typeAndSubTypes);
 
-        addClassificationNameConditionIfNecessary(query);
-
-        allGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
+        filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes,
query);
 
         if (context.getSearchParameters().getExcludeDeletedEntities()) {
-            allGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
+            filterGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
         }
     }
 
@@ -128,15 +137,15 @@ public class EntitySearchProcessor extends SearchProcessor {
 
                     vertices = getVerticesFromIndexQueryResult(queryResult);
 
-                    if (partialGraphQuery != null) {
+                    if (graphQuery != null) {
                         AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY,
getGuids(vertices));
 
-                        guidQuery.addConditionsFrom(partialGraphQuery);
+                        guidQuery.addConditionsFrom(graphQuery);
 
                         vertices = getVertices(guidQuery.vertices().iterator());
                     }
                 } else {
-                    Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset,
limit).iterator();
+                    Iterator<AtlasVertex> queryResult = graphQuery.vertices(qryOffset,
limit).iterator();
 
                     if (!queryResult.hasNext()) { // no more results from query - end of
search
                         break;
@@ -182,7 +191,7 @@ public class EntitySearchProcessor extends SearchProcessor {
 
         AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY,
getGuids(entityVertices));
 
-        query.addConditionsFrom(allGraphQuery);
+        query.addConditionsFrom(filterGraphQuery);
 
         List<AtlasVertex> ret = getVertices(query.vertices().iterator());
 
@@ -194,10 +203,4 @@ public class EntitySearchProcessor extends SearchProcessor {
 
         return ret;
     }
-
-    private void addClassificationNameConditionIfNecessary(AtlasGraphQuery query) {
-        if (context.getClassificationType() != null && !context.needClassificationProcessor())
{
-            query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, context.getClassificationType().getTypeAndAllSubTypes());
-        }
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 55a07f3..8dd7667 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -109,7 +109,7 @@ public class SearchContext {
     }
 
     public boolean needClassificationProcessor() {
-        return classificationType != null && (hasAttributeFilter(searchParameters.getTagFilters())
|| entityType == null);
+        return classificationType != null;
     }
 
     public boolean needEntityProcessor() {

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 1481809..ff0bd2e 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -88,7 +88,7 @@ public abstract class SearchProcessor {
     public abstract List<AtlasVertex> execute();
 
     public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
-        return nextProcessor == null ? entityVertices : nextProcessor.filter(entityVertices);
+        return nextProcessor == null || CollectionUtils.isEmpty(entityVertices) ? entityVertices
: nextProcessor.filter(entityVertices);
     }
 
 
@@ -193,12 +193,20 @@ public abstract class SearchProcessor {
             String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes, 0);
 
             if (StringUtils.isNotEmpty(filterQuery)) {
-                solrQuery.append(AND_STR).append(filterQuery);
+                if (solrQuery.length() > 0) {
+                    solrQuery.append(AND_STR);
+                }
+
+                solrQuery.append(filterQuery);
             }
         }
 
         if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities())
{
-            solrQuery.append(AND_STR).append("v.\"__state\":").append("ACTIVE");
+            if (solrQuery.length() > 0) {
+                solrQuery.append(AND_STR);
+            }
+
+            solrQuery.append("v.\"__state\":").append("ACTIVE");
         }
     }
 


Mime
View raw message