atlas-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject incubator-atlas git commit: ATLAS-1818: basic-search enhancements to improve search performance
Date Fri, 19 May 2017 19:13:14 GMT
Repository: incubator-atlas
Updated Branches:
  refs/heads/0.8-incubating 7bf728b8e -> a00a57be4


ATLAS-1818: basic-search enhancements to improve search performance

Signed-off-by: Madhan Neethiraj <madhan@apache.org>
(cherry picked from commit 04451449dd18f6c5df93fb4517c3d7fbdc122da7)


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/a00a57be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/a00a57be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/a00a57be

Branch: refs/heads/0.8-incubating
Commit: a00a57be49ed0523297e2ee16e92166409686b16
Parents: 7bf728b
Author: ashutoshm <amestry@hortonworks.com>
Authored: Fri May 19 10:26:37 2017 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Fri May 19 12:13:08 2017 -0700

----------------------------------------------------------------------
 distro/src/conf/atlas-application.properties    |   2 +
 distro/src/conf/solr/solrconfig.xml             |  22 +--
 .../atlas/repository/graphdb/AtlasGraph.java    |  14 ++
 .../repository/graphdb/titan0/Titan0Graph.java  |  71 +++++----
 .../test/resources/atlas-application.properties |   2 +
 .../repository/graphdb/titan1/Titan1Graph.java  |  55 ++++---
 .../atlas/discovery/EntityDiscoveryService.java | 158 ++++++++++++++-----
 .../services/EntityDiscoveryServiceTest.java    | 124 +++++++++++++++
 .../test/resources/atlas-application.properties |   2 +-
 9 files changed, 337 insertions(+), 113 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index b2b8e74..5e59528 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -62,6 +62,8 @@ atlas.graph.index.search.backend=${titan.index.backend}
 
 ${titan.index.properties}
 
+# Solr-specific configuration property
+atlas.graph.index.search.max-result-set-size=150
 
 #########  Notification Configs  #########
 atlas.notification.embedded=true

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/distro/src/conf/solr/solrconfig.xml
----------------------------------------------------------------------
diff --git a/distro/src/conf/solr/solrconfig.xml b/distro/src/conf/solr/solrconfig.xml
index ce2e20b..1d414f7 100644
--- a/distro/src/conf/solr/solrconfig.xml
+++ b/distro/src/conf/solr/solrconfig.xml
@@ -277,19 +277,19 @@
                and old cache.  
       -->
     <filterCache class="solr.FastLRUCache"
-                 size="512"
-                 initialSize="512"
-                 autowarmCount="0"/>
+                 size="2000"
+                 initialSize="2000"
+                 autowarmCount="1000"/>
 
     <!-- Query Result Cache
          
          Caches results of searches - ordered lists of document ids
          (DocList) based on a query, a sort, and the range of documents requested.  
       -->
-    <queryResultCache class="solr.LRUCache"
-                     size="512"
-                     initialSize="512"
-                     autowarmCount="0"/>
+    <queryResultCache class="solr.FastLRUCache"
+                      size="26000"
+                      initialSize="26000"
+                      autowarmCount="400"/>
    
     <!-- Document Cache
 
@@ -297,10 +297,10 @@
          document).  Since Lucene internal document ids are transient,
          this cache will not be autowarmed.  
       -->
-    <documentCache class="solr.LRUCache"
-                   size="512"
-                   initialSize="512"
-                   autowarmCount="0"/>
+    <documentCache class="solr.FastLRUCache"
+                   size="26000"
+                   initialSize="26000"
+                   autowarmCount="400"/>
     
     <!-- custom cache currently used by block join --> 
     <cache name="perSegFilter"

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
index a3a27bf..dded76f 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
@@ -158,6 +158,20 @@ public interface AtlasGraph<V, E> {
     AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString);
 
     /**
+     * Creates an index query.
+     *
+     * @param indexName index name
+     * @param queryString the query
+     * @param offset specify the offset that should be applied for the query. This is useful
for paging through
+     *               list of results
+     *
+     * @see <a
+     * href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">
+     * Elastic Search Reference</a> for query syntax
+     */
+    AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString, int offset);
+
+    /**
      * Gets the management object associated with this graph and opens a transaction
      * for changes that are made.
      * @return

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
index 9624c99..2408287 100644
--- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
@@ -17,39 +17,6 @@
  */
 package org.apache.atlas.repository.graphdb.titan0;
 
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.script.Bindings;
-import javax.script.ScriptContext;
-import javax.script.ScriptEngine;
-import javax.script.ScriptEngineManager;
-import javax.script.ScriptException;
-
-import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.groovy.GroovyExpression;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
-import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
-import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.repository.graphdb.GremlinVersion;
-import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
-import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
-import org.apache.atlas.typesystem.types.IDataType;
-
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
@@ -65,9 +32,40 @@ import com.tinkerpop.blueprints.Element;
 import com.tinkerpop.blueprints.Vertex;
 import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
 import com.tinkerpop.pipes.util.structures.Row;
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.GremlinVersion;
+import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
+import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
+import org.apache.atlas.typesystem.types.IDataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.script.Bindings;
+import javax.script.ScriptContext;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 
 /**
  * Titan 0.5.4 implementation of AtlasGraph.
@@ -163,7 +161,12 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge>
{
 
     @Override
     public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex,
String graphQuery) {
-        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery);
+        return indexQuery(fulltextIndex, graphQuery, 0);
+    }
+
+    @Override
+    public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex,
String graphQuery, int offset) {
+        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).offset(offset);
         return new Titan0IndexQuery(this, query);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/graphdb/titan0/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/resources/atlas-application.properties b/graphdb/titan0/src/test/resources/atlas-application.properties
index 636a9ff..3058330 100644
--- a/graphdb/titan0/src/test/resources/atlas-application.properties
+++ b/graphdb/titan0/src/test/resources/atlas-application.properties
@@ -50,6 +50,8 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 atlas.graph.index.search.solr.mode=cloud
 atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
 
+# Solr-specific configuration property
+atlas.graph.index.search.max-result-set-size=150
 
 #########  Hive Lineage Configs  #########
 # This models reflects the base super types for Data and Process

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java
----------------------------------------------------------------------
diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java
b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java
index 6a61075..e829d91 100644
--- a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java
+++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1Graph.java
@@ -17,19 +17,18 @@
  */
 package org.apache.atlas.repository.graphdb.titan1;
 
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.script.Bindings;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.PropertyKey;
+import com.thinkaurelius.titan.core.SchemaViolationException;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanIndexQuery;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
+import com.thinkaurelius.titan.core.schema.TitanManagement;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.groovy.GroovyExpression;
@@ -57,18 +56,17 @@ import org.apache.tinkerpop.gremlin.structure.io.IoCore;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.thinkaurelius.titan.core.Cardinality;
-import com.thinkaurelius.titan.core.PropertyKey;
-import com.thinkaurelius.titan.core.SchemaViolationException;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanIndexQuery;
-import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
-import com.thinkaurelius.titan.core.schema.TitanManagement;
-import com.thinkaurelius.titan.core.util.TitanCleanup;
+import javax.script.Bindings;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Titan 1.0.0 implementation of AtlasGraph.
@@ -179,7 +177,12 @@ public class Titan1Graph implements AtlasGraph<Titan1Vertex, Titan1Edge>
{
 
     @Override
     public AtlasIndexQuery<Titan1Vertex, Titan1Edge> indexQuery(String fulltextIndex,
String graphQuery) {
-        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery);
+        return indexQuery(fulltextIndex, graphQuery, 0);
+    }
+
+    @Override
+    public AtlasIndexQuery<Titan1Vertex, Titan1Edge> indexQuery(String fulltextIndex,
String graphQuery, int offset) {
+        TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery).offset(offset);
         return new Titan1IndexQuery(this, query);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index e6a06c3..1b4583a 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -17,14 +17,16 @@
  */
 package org.apache.atlas.discovery;
 
+import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasConfiguration;
 import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
-import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
-import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
+import org.apache.atlas.AtlasException;
 import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
+import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
+import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
 import org.apache.atlas.model.instance.AtlasEntity.Status;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.query.Expressions.AliasExpression;
@@ -83,14 +85,20 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
     private final EntityGraphRetriever            entityRetriever;
     private final AtlasGremlinQueryProvider       gremlinQueryProvider;
     private final AtlasTypeRegistry               typeRegistry;
+    private final int                             maxResultSetSize;
+    private final int                             maxTypesCountInIdxQuery;
+    private final int                             maxTagsCountInIdxQuery;
 
     @Inject
-    EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry)
{
+    EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry)
throws AtlasException {
         this.graph                    = AtlasGraphProvider.getGraphInstance();
         this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
         this.entityRetriever          = new EntityGraphRetriever(typeRegistry);
         this.gremlinQueryProvider     = AtlasGremlinQueryProvider.INSTANCE;
         this.typeRegistry             = typeRegistry;
+        this.maxResultSetSize         = ApplicationProperties.get().getInt("atlas.graph.index.search.max-result-set-size",
150);
+        this.maxTypesCountInIdxQuery  = ApplicationProperties.get().getInt("atlas.graph.index.search.max-types-count",
10);
+        this.maxTagsCountInIdxQuery   = ApplicationProperties.get().getInt("atlas.graph.index.search.max-tags-count",
10);
     }
 
     @Override
@@ -217,7 +225,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
                     if (attribute == null) {
                         throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName,
typeName);
                     }
-                    
+
                 } else {
                     // if attrName is null|empty iterate defaultAttrNames to get attribute
value
                     final List<String> defaultAttrNames = new ArrayList<>(Arrays.asList("qualifiedName",
"name"));
@@ -238,7 +246,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
                 } else {
                     attrQualifiedName = attribute.getQualifiedName();
 
-                    String  attrQuery = String.format("%s AND (%s *)", attrName, attrValuePrefix.replaceAll("\\.",
" "));
+                    String attrQuery = String.format("%s AND (%s *)", attrName, attrValuePrefix.replaceAll("\\.",
" "));
 
                     query = StringUtils.isEmpty(query) ? attrQuery : String.format("(%s)
AND (%s)", query, attrQuery);
                 }
@@ -252,59 +260,72 @@ public class EntityDiscoveryService implements AtlasDiscoveryService
{
         // if query was provided, perform indexQuery and filter for typeName & classification
in memory; this approach
         // results in a faster and accurate results than using CONTAINS/CONTAINS_PREFIX filter
on entityText property
         if (StringUtils.isNotEmpty(query)) {
-            final String                idxQuery   = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY,
query);
-            final Iterator<Result<?,?>> qryResult  = graph.indexQuery(Constants.FULLTEXT_INDEX,
idxQuery).vertices();
-            final int                   startIdx   = params.offset();
-            final int                   resultSize = params.limit();
+            final String idxQuery   = getQueryForFullTextSearch(query, typeName, classification);
+            final int    startIdx   = params.offset();
+            final int    resultSize = params.limit();
+            int          resultIdx  = 0;
 
-            int resultIdx = 0;
+            for (int indexQueryOffset = 0; ; indexQueryOffset += getMaxResultSetSize()) {
+                final Iterator<Result<?, ?>> qryResult = graph.indexQuery(Constants.FULLTEXT_INDEX,
idxQuery, indexQueryOffset).vertices();
 
-            while (qryResult.hasNext()) {
-                AtlasVertex<?,?> vertex = qryResult.next().getVertex();
-
-                String vertexTypeName = GraphHelper.getTypeName(vertex);
-
-                // skip non-entity vertices
-                if (StringUtils.isEmpty(vertexTypeName) || StringUtils.isEmpty(GraphHelper.getGuid(vertex)))
{
-                    continue;
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("indexQuery: query=" + idxQuery + "; offset=" + indexQueryOffset);
                 }
 
-                if (typeNames != null && !typeNames.contains(vertexTypeName)) {
-                    continue;
+                if(!qryResult.hasNext()) {
+                    break;
                 }
 
-                if (classificationNames != null) {
-                    List<String> traitNames = GraphHelper.getTraitNames(vertex);
+                while (qryResult.hasNext()) {
+                    AtlasVertex<?, ?> vertex         = qryResult.next().getVertex();
+                    String            vertexTypeName = GraphHelper.getTypeName(vertex);
 
-                    if (CollectionUtils.isEmpty(traitNames) ||
-                        !CollectionUtils.containsAny(classificationNames, traitNames)) {
+                    // skip non-entity vertices
+                    if (StringUtils.isEmpty(vertexTypeName) || StringUtils.isEmpty(GraphHelper.getGuid(vertex)))
{
+                        continue;
+                    }
+
+                    if (typeNames != null && !typeNames.contains(vertexTypeName))
{
                         continue;
                     }
-                }
 
-                if (isAttributeSearch) {
-                    String vertexAttrValue = vertex.getProperty(attrQualifiedName, String.class);
+                    if (classificationNames != null) {
+                        List<String> traitNames = GraphHelper.getTraitNames(vertex);
 
-                    if (StringUtils.isNotEmpty(vertexAttrValue) && !vertexAttrValue.startsWith(attrValuePrefix))
{
+                        if (CollectionUtils.isEmpty(traitNames) ||
+                                !CollectionUtils.containsAny(classificationNames, traitNames))
{
+                            continue;
+                        }
+                    }
+
+                    if (isAttributeSearch) {
+                        String vertexAttrValue = vertex.getProperty(attrQualifiedName, String.class);
+
+                        if (StringUtils.isNotEmpty(vertexAttrValue) && !vertexAttrValue.startsWith(attrValuePrefix))
{
+                            continue;
+                        }
+                    }
+
+                    if (skipDeletedEntities(excludeDeletedEntities, vertex)) {
                         continue;
                     }
-                }
 
-                if (skipDeletedEntities(excludeDeletedEntities, vertex)) {
-                    continue;
-                }
+                    resultIdx++;
 
-                resultIdx++;
+                    if (resultIdx <= startIdx) {
+                        continue;
+                    }
 
-                if (resultIdx <= startIdx) {
-                    continue;
-                }
+                    AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex);
 
-                AtlasEntityHeader header = entityRetriever.toAtlasEntityHeader(vertex);
+                    ret.addEntity(header);
 
-                ret.addEntity(header);
+                    if (ret.getEntities().size() == resultSize) {
+                        break;
+                    }
+                }
 
-                if (ret.getEntities().size() == resultSize) {
+                if (ret.getEntities() != null && ret.getEntities().size() == resultSize)
{
                     break;
                 }
             }
@@ -347,7 +368,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
                 Object result = graph.executeGremlinScript(scriptEngine, bindings, basicQuery,
false);
 
                 if (result instanceof List && CollectionUtils.isNotEmpty((List) result))
{
-                    List   queryResult  = (List) result;
+                    List queryResult = (List) result;
                     Object firstElement = queryResult.get(0);
 
                     if (firstElement instanceof AtlasVertex) {
@@ -371,6 +392,57 @@ public class EntityDiscoveryService implements AtlasDiscoveryService
{
         return ret;
     }
 
+    private String getQueryForFullTextSearch(String userKeyedString, String typeName, String
classification) {
+        String typeFilter          = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
+        String classficationFilter = getClassificationFilter(typeRegistry, classification,
maxTagsCountInIdxQuery);
+
+        StringBuilder queryText = new StringBuilder();
+
+        if (! StringUtils.isEmpty(userKeyedString)) {
+            queryText.append(userKeyedString);
+        }
+
+        if (! StringUtils.isEmpty(typeFilter)) {
+            if (queryText.length() > 0) {
+                queryText.append(" AND ");
+            }
+
+            queryText.append(typeFilter);
+        }
+
+        if (! StringUtils.isEmpty(classficationFilter)) {
+            if (queryText.length() > 0) {
+                queryText.append(" AND ");
+            }
+
+            queryText.append(classficationFilter);
+        }
+
+        return String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, queryText.toString());
+    }
+
+    private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String
classificationName, int maxTypesCountInIdxQuery) {
+        AtlasClassificationType classification  = typeRegistry.getClassificationTypeByName(classificationName);
+        Set<String>             typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes()
: null;
+
+        if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size()
<= maxTypesCountInIdxQuery) {
+            return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+        }
+
+        return "";
+    }
+
+    private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName,
int maxTypesCountInIdxQuery) {
+        AtlasEntityType type            = typeRegistry.getEntityTypeByName(typeName);
+        Set<String>     typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes()
: null;
+
+        if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size()
<= maxTypesCountInIdxQuery) {
+            return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+        }
+
+        return "";
+    }
+
     private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams
params, boolean excludeDeletedEntities) throws AtlasBaseException {
         List<AtlasFullTextResult> ret  = new ArrayList<>();
         Iterator<Result>          iter = query.vertices();
@@ -471,4 +543,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
     private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex<?,
?> vertex) {
         return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
     }
+
+    public int getMaxResultSetSize() {
+        return maxResultSetSize;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
b/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
new file mode 100644
index 0000000..d503ef7
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.services;
+
+import org.apache.atlas.TestOnlyModule;
+import org.apache.atlas.discovery.EntityDiscoveryService;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
+import org.powermock.reflect.Whitebox;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.testng.Assert.assertEquals;
+
+@Guice(modules = TestOnlyModule.class)
+public class EntityDiscoveryServiceTest {
+
+    private final String TEST_TYPE                = "test";
+    private final String TEST_TYPE1               = "test1";
+    private final String TEST_TYPE2               = "test2";
+    private final String TEST_TYPE3               = "test3";
+    private final String TEST_TYPE_WITH_SUB_TYPES = "testTypeWithSubTypes";
+    private AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
+
+    AtlasEntityDef typeTest         = null;
+    AtlasEntityDef typeTest1        = null;
+    AtlasEntityDef typeTest2        = null;
+    AtlasEntityDef typeTest3        = null;
+    AtlasEntityDef typeWithSubTypes = null;
+
+    private final int maxTypesCountInIdxQuery = 10;
+
+
+    @BeforeClass
+    public void init() throws AtlasBaseException {
+        typeTest         = new AtlasEntityDef(TEST_TYPE);
+        typeTest1        = new AtlasEntityDef(TEST_TYPE1);
+        typeTest2        = new AtlasEntityDef(TEST_TYPE2);
+        typeTest3        = new AtlasEntityDef(TEST_TYPE3);
+        typeWithSubTypes = new AtlasEntityDef(TEST_TYPE_WITH_SUB_TYPES);
+
+        typeTest1.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
+        typeTest2.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
+        typeTest3.addSuperType(TEST_TYPE_WITH_SUB_TYPES);
+
+        AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate();
+
+        ttr.addType(typeTest);
+        ttr.addType(typeWithSubTypes);
+        ttr.addType(typeTest1);
+        ttr.addType(typeTest2);
+        ttr.addType(typeTest3);
+
+        typeRegistry.releaseTypeRegistryForUpdate(ttr, true);
+    }
+
+    @Test
+    public void getSubTypesForType_NullStringReturnsEmptyString() throws Exception {
+        invokeGetSubTypesForType(null, maxTypesCountInIdxQuery);
+    }
+
+    @Test
+    public void getSubTypesForType_BlankStringReturnsEmptyString() throws Exception {
+        invokeGetSubTypesForType(" ", maxTypesCountInIdxQuery);
+    }
+
+    @Test
+    public void getSubTypesForType_EmptyStringReturnsEmptyString() throws Exception {
+        invokeGetSubTypesForType("", maxTypesCountInIdxQuery);
+    }
+
+    @Test
+    public void getSubTypeForTypeWithNoSubType_ReturnsTypeString() throws Exception {
+        String s = invokeGetSubTypesForType(TEST_TYPE, 10);
+
+        assertEquals(s, "(" + TEST_TYPE + ")");
+    }
+
+    @Test
+    public void getSubTypeForTypeWithSubTypes_ReturnsOrClause() throws Exception {
+        String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, maxTypesCountInIdxQuery);
+
+        assertTrue(s.startsWith("(" + TEST_TYPE_WITH_SUB_TYPES));
+        assertTrue(s.contains(" " + TEST_TYPE1));
+        assertTrue(s.contains(" " + TEST_TYPE2));
+        assertTrue(s.contains(" " + TEST_TYPE3));
+        assertTrue(s.endsWith(")"));
+    }
+
+    @Test
+    public void getSubTypeForTypeWithSubTypes_ReturnsEmptyString() throws Exception {
+        String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, 2);
+
+        assertTrue(StringUtils.isBlank(s));
+    }
+
+    private String invokeGetSubTypesForType(String inputString, int maxSubTypes) throws Exception
{
+        String s = Whitebox.invokeMethod(EntityDiscoveryService.class, "getTypeFilter", typeRegistry,
inputString, maxSubTypes);
+
+        assertNotNull(s);
+        return s;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/a00a57be/typesystem/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/typesystem/src/test/resources/atlas-application.properties b/typesystem/src/test/resources/atlas-application.properties
index 5ffde5e..c4ce5ea 100644
--- a/typesystem/src/test/resources/atlas-application.properties
+++ b/typesystem/src/test/resources/atlas-application.properties
@@ -72,7 +72,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
 # Solr cloud mode properties
 atlas.graph.index.search.solr.mode=cloud
 atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
-
+atlas.graph.index.search.max-result-set-size=150
 
 #########  Hive Lineage Configs  #########
 ## Schema


Mime
View raw message