atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [23/39] atlas git commit: ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 2)
Date Mon, 24 Jul 2017 15:57:28 GMT
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 2)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/cfb6b84f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/cfb6b84f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/cfb6b84f

Branch: refs/heads/feature-odf
Commit: cfb6b84f41b05275db826bbd43e4c39145b6d2d5
Parents: 377fe19
Author: Madhan Neethiraj <madhan@apache.org>
Authored: Tue Jul 18 13:27:46 2017 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Tue Jul 18 16:59:02 2017 -0700

----------------------------------------------------------------------
 .../atlas/discovery/EntitySearchProcessor.java  | 30 +++++++++++-----
 .../discovery/FullTextSearchProcessor.java      | 36 ++++++++++++++++++--
 .../apache/atlas/discovery/SearchContext.java   |  9 +++--
 .../apache/atlas/discovery/SearchProcessor.java |  4 +--
 4 files changed, 61 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 3204ecf..50376ef 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
 import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
 import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.type.AtlasClassificationType;
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.commons.collections.CollectionUtils;
@@ -39,18 +40,21 @@ public class EntitySearchProcessor extends SearchProcessor {
     public EntitySearchProcessor(SearchContext context) {
         super(context);
 
-        final AtlasEntityType entityType         = context.getEntityType();
-        final FilterCriteria  filterCriteria     = context.getSearchParameters().getEntityFilters();
-        final Set<String>     typeAndSubTypes    = entityType.getTypeAndAllSubTypes();
-        final Set<String>     solrAttributes     = new HashSet<>();
-        final Set<String>     gremlinAttributes  = new HashSet<>();
-        final Set<String>     allAttributes      = new HashSet<>();
+        final AtlasEntityType entityType        = context.getEntityType();
+        final FilterCriteria  filterCriteria    = context.getSearchParameters().getEntityFilters();
+        final Set<String>     typeAndSubTypes   = entityType.getTypeAndAllSubTypes();
+        final Set<String>     solrAttributes    = new HashSet<>();
+        final Set<String>     gremlinAttributes = new HashSet<>();
+        final Set<String>     allAttributes     = new HashSet<>();
+
+        final AtlasClassificationType classificationType   = context.getClassificationType();
+        final boolean                 filterClassification = classificationType != null &&
!context.needClassificationProcessor();
 
 
         processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes,
allAttributes);
 
-        final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
-        final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false);
+        final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size()
<= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+        final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes)
&& canApplySolrFilter(entityType, filterCriteria, false);
 
         StringBuilder solrQuery = new StringBuilder();
 
@@ -82,6 +86,10 @@ public class EntitySearchProcessor extends SearchProcessor {
                 query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
             }
 
+            if (filterClassification) {
+                query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
+            }
+
             graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes,
query);
 
             if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery
== null) {
@@ -93,6 +101,10 @@ public class EntitySearchProcessor extends SearchProcessor {
 
         AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY,
typeAndSubTypes);
 
+        if (filterClassification) {
+            query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
+        }
+
         filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes,
query);
 
         if (context.getSearchParameters().getExcludeDeletedEntities()) {
@@ -115,7 +127,7 @@ public class EntitySearchProcessor extends SearchProcessor {
         }
 
         try {
-            int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset()
: 0;
+            int qryOffset = (nextProcessor == null && (graphQuery == null || indexQuery
== null)) ? context.getSearchParameters().getOffset() : 0;
             int limit     = context.getSearchParameters().getLimit();
             int resultIdx = qryOffset;
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 4ddd642..83368c2 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -22,12 +22,14 @@ import org.apache.atlas.repository.Constants;
 import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
 import org.apache.atlas.repository.graphdb.AtlasVertex;
 import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
 
 
 public class FullTextSearchProcessor extends SearchProcessor {
@@ -40,9 +42,39 @@ public class FullTextSearchProcessor extends SearchProcessor {
         super(context);
 
         SearchParameters searchParameters = context.getSearchParameters();
-        String           queryString      = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY,
searchParameters.getQuery());
+        StringBuilder    queryString      = new StringBuilder();
 
-        indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString);
+        queryString.append("v.\"").append(Constants.ENTITY_TEXT_PROPERTY_KEY).append("\":(").append(searchParameters.getQuery());
+
+        // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
+        // processing (and rejection) by subsequent EntitySearchProcessor
+        if (context.getEntityType() != null) {
+            Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes();
+
+            if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) {
+                queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
SPACE_STRING)).append(")");
+            } else {
+                LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might
cause poor performance",
+                         context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+            }
+        }
+
+        // if search includes classification criteria, adding a filter here can help avoid
unnecessary
+        // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
+        if (context.getClassificationType() != null) {
+            Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes();
+
+            if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY)
{
+                queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames,
SPACE_STRING)).append(")");
+            } else {
+                LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might
cause poor performance",
+                        context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+            }
+        }
+
+        queryString.append(")");
+
+        indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString.toString());
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 8dd7667..929f8d0 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -61,7 +61,6 @@ public class SearchContext {
 
         if (needEntityProcessor()) {
             addProcessor(new EntitySearchProcessor(this));
-
         }
     }
 
@@ -104,15 +103,15 @@ public class SearchContext {
         return toString(new StringBuilder()).toString();
     }
 
-    public boolean needFullTextrocessor() {
+    boolean needFullTextrocessor() {
         return StringUtils.isNotEmpty(searchParameters.getQuery());
     }
 
-    public boolean needClassificationProcessor() {
-        return classificationType != null;
+    boolean needClassificationProcessor() {
+        return classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()));
     }
 
-    public boolean needEntityProcessor() {
+    boolean needEntityProcessor() {
         return entityType != null;
     }
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index ff0bd2e..596b43b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -181,7 +181,7 @@ public abstract class SearchProcessor {
     protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes)
{
         String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
 
-        solrQuery.append("v.\"__typeName\": (")
+        solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\": (")
                 .append(typeAndSubtypesString)
                 .append(")");
     }
@@ -206,7 +206,7 @@ public abstract class SearchProcessor {
                 solrQuery.append(AND_STR);
             }
 
-            solrQuery.append("v.\"__state\":").append("ACTIVE");
+            solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
         }
     }
 


Mime
View raw message