Mailing-List: contact commits-help@atlas.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@atlas.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: madhan@apache.org
To: commits@atlas.apache.org
Date: Mon, 24 Jul 2017 15:57:18 -0000
Message-Id: <54a7b1967e134d5a8cffb0ef0d661a03@git.apache.org>
In-Reply-To: <b4446226b4a3488886c44cc599965ba4@git.apache.org>
References: <b4446226b4a3488886c44cc599965ba4@git.apache.org>
Subject: [13/39] atlas git commit: ATLAS-1947: AtlasSearchResult to include
 referredEntity headers
archived-at: Mon, 24 Jul 2017 15:57:14 -0000

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java b/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java
deleted file mode 100644
index 0f91b2d..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasStructType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.util.SearchTracker;
-import org.apache.atlas.utils.AtlasPerfTracer;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.builder.ToStringBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-@Component
-public class SearchPipeline {
-    private static final Logger LOG      = LoggerFactory.getLogger(SearchPipeline.class);
-    private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("SearchPipeline");
-
-    enum ExecutionMode { SOLR, GREMLIN, MIXED }
-
-    enum IndexResultType { TAG, ENTITY, TEXT }
-
-    private final SolrStep                 solrStep;
-    private final GremlinStep              gremlinStep;
-    private final SearchTracker            searchTracker;
-    private final AtlasTypeRegistry        typeRegistry;
-    private final Configuration            atlasConfiguration;
-    private final GraphBackedSearchIndexer indexer;
-
-    @Inject
-    public SearchPipeline(SolrStep solrStep, GremlinStep gremlinStep, SearchTracker searchTracker, AtlasTypeRegistry typeRegistry, Configuration atlasConfiguration, GraphBackedSearchIndexer indexer) {
-        this.solrStep           = solrStep;
-        this.gremlinStep        = gremlinStep;
-        this.searchTracker      = searchTracker;
-        this.typeRegistry       = typeRegistry;
-        this.atlasConfiguration = atlasConfiguration;
-        this.indexer            = indexer;
-    }
-
-    public List<AtlasVertex> run(SearchParameters searchParameters) throws AtlasBaseException {
-        final List<AtlasVertex> ret;
-
-        AtlasPerfTracer perf = null;
-
-        if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
-            perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "SearchPipeline.run("+ searchParameters +")");
-        }
-
-        AtlasEntityType         entityType = typeRegistry.getEntityTypeByName(searchParameters.getTypeName());
-        AtlasClassificationType classiType = typeRegistry.getClassificationTypeByName(searchParameters.getClassification());
-        PipelineContext         context    = new PipelineContext(searchParameters, entityType, classiType, indexer.getVertexIndexKeys());
-        String                  searchId   = searchTracker.add(context); // For future cancellation
-
-        try {
-            ExecutionMode mode = determineExecutionMode(context);
-
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("Execution mode {}", mode);
-            }
-
-            switch (mode) {
-                case SOLR:
-                    ret = runOnlySolr(context);
-                    break;
-
-                case GREMLIN:
-                    ret = runOnlyGremlin(context);
-                    break;
-
-                case MIXED:
-                    ret = runMixed(context);
-                    break;
-
-                default:
-                    ret = Collections.emptyList();
-            }
-        } finally {
-            searchTracker.remove(searchId);
-
-            AtlasPerfTracer.log(perf);
-        }
-
-        return ret;
-    }
-
-    private List<AtlasVertex> runOnlySolr(PipelineContext context) throws AtlasBaseException {
-        // Only when there's no tag and query
-        List<AtlasVertex> results = new ArrayList<>();
-
-        while (results.size() < context.getSearchParameters().getLimit()) {
-            if (context.getForceTerminate()) {
-                LOG.debug("search has been terminated");
-
-                break;
-            }
-
-            // Execute solr search only
-            solrStep.execute(context);
-
-            List<AtlasVertex> stepResults = getIndexResults(context);
-
-            context.incrementSearchRound();
-
-            addToResult(results, stepResults, context.getSearchParameters().getLimit());
-
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("Pipeline iteration {}: stepResults={}; totalResult={}", context.getIterationCount(), stepResults.size(), results.size());
-            }
-
-            if (CollectionUtils.isEmpty(stepResults)) {
-                // If no result is found any subsequent iteration then just stop querying the index
-                break;
-            }
-        }
-
-        if (context.getIndexResultType() == IndexResultType.TAG) {
-            List<AtlasVertex> entityVertices = new ArrayList<>(results.size());
-
-            for (AtlasVertex tagVertex : results) {
-                Iterable<AtlasEdge> edges = tagVertex.getEdges(AtlasEdgeDirection.IN);
-
-                for (AtlasEdge edge : edges) {
-                    AtlasVertex entityVertex = edge.getOutVertex();
-
-                    entityVertices.add(entityVertex);
-                }
-            }
-
-            results = entityVertices;
-        }
-
-        return results;
-    }
-
-    private List<AtlasVertex> runOnlyGremlin(PipelineContext context) throws AtlasBaseException {
-        List<AtlasVertex> results = new ArrayList<>();
-
-        while (results.size() < context.getSearchParameters().getLimit()) {
-            if (context.getForceTerminate()) {
-                LOG.debug("search has been terminated");
-
-                break;
-            }
-
-            gremlinStep.execute(context);
-
-            List<AtlasVertex> stepResults = getGremlinResults(context);
-
-            context.incrementSearchRound();
-
-            addToResult(results, stepResults, context.getSearchParameters().getLimit());
-
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("Pipeline iteration {}: stepResults={}; totalResult={}", context.getIterationCount(), stepResults.size(), results.size());
-            }
-
-            if (CollectionUtils.isEmpty(stepResults)) {
-                // If no result is found any subsequent iteration then just stop querying the index
-                break;
-            }
-        }
-
-        return results;
-    }
-
-    /*
-        1. Index processes few attributes and then gremlin processes rest
-            1.1 Iterate for gremlin till the index results are non null
-        2. Index processes all attributes, gremlin has nothing to do
-
-        Sometimes the result set might be less than the max limit and we need to iterate until the result set is full
-        or the iteration doesn't return any results
-
-     */
-    private List<AtlasVertex> runMixed(PipelineContext context) throws AtlasBaseException {
-        List<AtlasVertex> results = new ArrayList<>();
-
-        while (results.size() < context.getSearchParameters().getLimit()) {
-            if (context.getForceTerminate()) {
-                LOG.debug("search has been terminated");
-
-                break;
-            }
-
-            // Execute Solr search and then pass it to the Gremlin step (if needed)
-            solrStep.execute(context);
-
-            if (!context.hasIndexResults()) {
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("No index results in iteration {}", context.getIterationCount());
-                }
-
-                // If no result is found any subsequent iteration then just stop querying the index
-                break;
-            }
-
-            // Attributes partially processed by Solr, use gremlin to process remaining attribute(s)
-            gremlinStep.execute(context);
-
-            context.incrementSearchRound();
-
-            List<AtlasVertex> stepResults = getGremlinResults(context);
-
-            addToResult(results, stepResults, context.getSearchParameters().getLimit());
-
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("Pipeline iteration {}: stepResults={}; totalResult={}", context.getIterationCount(), stepResults.size(), results.size());
-            }
-        }
-
-        return results;
-    }
-
-    private void addToResult(List<AtlasVertex> result, List<AtlasVertex> stepResult, int maxLimit) {
-        if (result != null && stepResult != null && result.size() < maxLimit) {
-            for (AtlasVertex vertex : stepResult) {
-                result.add(vertex);
-
-                if (result.size() >= maxLimit) {
-                    break;
-                }
-            }
-        }
-    }
-
-    private List<AtlasVertex> getIndexResults(PipelineContext pipelineContext) {
-        List<AtlasVertex> ret = new ArrayList<>();
-
-        if (pipelineContext.hasIndexResults()) {
-            Iterator<AtlasIndexQuery.Result> iter = pipelineContext.getIndexResultsIterator();
-
-            while(iter.hasNext()) {
-                ret.add(iter.next().getVertex());
-            }
-        }
-
-        return ret;
-    }
-
-    private List<AtlasVertex> getGremlinResults(PipelineContext pipelineContext) {
-        List<AtlasVertex> ret = new ArrayList<>();
-
-        if (pipelineContext.hasGremlinResults()) {
-            Iterator<AtlasVertex> iter = pipelineContext.getGremlinResultIterator();
-
-            while (iter.hasNext()) {
-                ret.add(iter.next());
-            }
-        }
-
-        return ret;
-    }
-
-    private ExecutionMode determineExecutionMode(PipelineContext context) {
-        SearchParameters        searchParameters   = context.getSearchParameters();
-        AtlasClassificationType classificationType = context.getClassificationType();
-        AtlasEntityType         entityType         = context.getEntityType();
-        int                     solrCount          = 0;
-        int                     gremlinCount       = 0;
-
-        if (StringUtils.isNotEmpty(searchParameters.getQuery())) {
-            solrCount++;
-
-            // __state index only exists in vertex_index
-            if (searchParameters.getExcludeDeletedEntities()) {
-                gremlinCount++;
-            }
-        }
-
-        if (classificationType != null) {
-            Set<String> typeAndAllSubTypes = classificationType.getTypeAndAllSubTypes();
-
-            if (typeAndAllSubTypes.size() > atlasConfiguration.getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10)) {
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("Classification type {} has too many subTypes ({}) to use in Solr search. Gremlin will be used to execute the search",
-                              classificationType.getTypeName(), typeAndAllSubTypes.size());
-                }
-
-                gremlinCount++;
-            } else {
-                if (hasNonIndexedAttrViolation(classificationType, context.getIndexedKeys(), searchParameters.getTagFilters())) {
-                    if (LOG.isDebugEnabled()) {
-                        LOG.debug("Tag filters not suitable for Solr search. Gremlin will be used to execute the search");
-                    }
-
-                    gremlinCount++;
-                } else {
-                    solrCount++;
-
-                    // __state index only exist in vertex_index
-                    if (searchParameters.getExcludeDeletedEntities()) {
-                        gremlinCount++;
-                    }
-                }
-            }
-        }
-
-        if (entityType != null) {
-            Set<String> typeAndAllSubTypes = entityType.getTypeAndAllSubTypes();
-
-            if (typeAndAllSubTypes.size() > atlasConfiguration.getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10)) {
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("Entity type {} has too many subTypes ({}) to use in Solr search. Gremlin will be used to execute the search",
-                              entityType.getTypeName(), typeAndAllSubTypes.size());
-                }
-
-                gremlinCount++;
-            } else {
-                if (hasNonIndexedAttrViolation(entityType, context.getIndexedKeys(), searchParameters.getEntityFilters())) {
-                    if (LOG.isDebugEnabled()) {
-                        LOG.debug("Entity filters not suitable for Solr search. Gremlin will be used to execute the search");
-                    }
-
-                    gremlinCount++;
-                } else {
-                    solrCount++;
-                }
-            }
-        }
-
-        ExecutionMode mode = ExecutionMode.MIXED;
-
-        if (solrCount == 1 && gremlinCount == 0) {
-            mode = ExecutionMode.SOLR;
-        } else if (gremlinCount == 1 && solrCount == 0) {
-            mode = ExecutionMode.GREMLIN;
-        }
-
-        return mode;
-    }
-
-    // If Index can't process all attributes and any of the non-indexed attribute is present in OR nested within AND
-    // then the only way is Gremlin
-    // A violation (here) is defined as presence of non-indexed attribute within any OR clause nested under an AND clause
-    // the reason being that the index would not be able to process the nested OR attribute which might result in
-    // exclusion of valid result (vertex)
-    private boolean hasNonIndexedAttrViolation(AtlasStructType structType, Set<String> indexKeys, FilterCriteria filterCriteria) {
-        return hasNonIndexedAttrViolation(structType, indexKeys, filterCriteria, false);
-    }
-
-    private boolean hasNonIndexedAttrViolation(AtlasStructType structType, Set<String> indexKeys, FilterCriteria filterCriteria, boolean enclosedInOrCondition) {
-        if (filterCriteria == null) {
-            return false;
-        }
-
-        boolean              ret             = false;
-        Condition            filterCondition = filterCriteria.getCondition();
-        List<FilterCriteria> criterion       = filterCriteria.getCriterion();
-
-        if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
-            if (!enclosedInOrCondition) {
-                enclosedInOrCondition = filterCondition == Condition.OR;
-            }
-
-            // If we have nested criterion let's find any nested ORs with non-indexed attr
-            for (FilterCriteria criteria : criterion) {
-                ret |= hasNonIndexedAttrViolation(structType, indexKeys, criteria, enclosedInOrCondition);
-
-                if (ret) {
-                    break;
-                }
-            }
-        } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
-            // If attribute qualified name doesn't exist in the vertex index we potentially might have a problem
-            try {
-                String qualifiedAttributeName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
-
-                ret = CollectionUtils.isEmpty(indexKeys) || !indexKeys.contains(qualifiedAttributeName);
-
-                if (ret) {
-                    LOG.warn("search includes non-indexed attribute '{}'; might cause poor performance", qualifiedAttributeName);
-                }
-            } catch (AtlasBaseException e) {
-                LOG.warn(e.getMessage());
-
-                ret = true;
-            }
-        }
-
-        // return ret && enclosedInOrCondition;
-
-        return ret;
-    }
-
-    public interface PipelineStep {
-        void execute(PipelineContext context) throws AtlasBaseException;
-    }
-
-    public static class PipelineContext {
-        // TODO: See if anything can be cached in the context
-
-        private final SearchParameters        searchParameters;
-        private final AtlasEntityType         entityType;
-        private final AtlasClassificationType classificationType;
-        private final Set<String>             indexedKeys;
-
-        private int     iterationCount;
-        private boolean forceTerminate;
-        private int     currentOffset;
-        private int     maxLimit;
-
-        // Continuous processing stuff
-        private Set<String> tagSearchAttributes       = new HashSet<>();
-        private Set<String> entitySearchAttributes    = new HashSet<>();
-        private Set<String> tagAttrProcessedBySolr    = new HashSet<>();
-        private Set<String> entityAttrProcessedBySolr = new HashSet<>();
-
-        // Results related stuff
-        private IndexResultType                  indexResultType;
-        private Iterator<AtlasIndexQuery.Result> indexResultsIterator;
-        private Iterator<AtlasVertex>            gremlinResultIterator;
-
-        private Map<String, AtlasIndexQuery> cachedIndexQueries = new HashMap<>();
-        private Map<String, AtlasGraphQuery> cachedGraphQueries = new HashMap<>();
-
-        public PipelineContext(SearchParameters searchParameters, AtlasEntityType entityType, AtlasClassificationType classificationType, Set<String> indexedKeys) {
-            this.searchParameters   = searchParameters;
-            this.entityType         = entityType;
-            this.classificationType = classificationType;
-            this.indexedKeys        = indexedKeys;
-
-            currentOffset = searchParameters.getOffset();
-            maxLimit      = searchParameters.getLimit();
-        }
-
-        public SearchParameters getSearchParameters() {
-            return searchParameters;
-        }
-
-        public AtlasEntityType getEntityType() {
-            return entityType;
-        }
-
-        public AtlasClassificationType getClassificationType() {
-            return classificationType;
-        }
-
-        public Set<String> getIndexedKeys() { return indexedKeys; }
-
-        public int getIterationCount() {
-            return iterationCount;
-        }
-
-        public boolean getForceTerminate() {
-            return forceTerminate;
-        }
-
-        public void setForceTerminate(boolean forceTerminate) {
-            this.forceTerminate = forceTerminate;
-        }
-
-        public boolean hasProcessedTagAttribute(String attributeName) {
-            return tagAttrProcessedBySolr.contains(attributeName);
-        }
-
-        public boolean hasProcessedEntityAttribute(String attributeName) {
-            return entityAttrProcessedBySolr.contains(attributeName);
-        }
-
-        public Iterator<AtlasIndexQuery.Result> getIndexResultsIterator() {
-            return indexResultsIterator;
-        }
-
-        public void setIndexResultsIterator(Iterator<AtlasIndexQuery.Result> indexResultsIterator) {
-            this.indexResultsIterator = indexResultsIterator;
-        }
-
-        public Iterator<AtlasVertex> getGremlinResultIterator() {
-            return gremlinResultIterator;
-        }
-
-        public void setGremlinResultIterator(Iterator<AtlasVertex> gremlinResultIterator) {
-            this.gremlinResultIterator = gremlinResultIterator;
-        }
-
-        public boolean hasIndexResults() {
-            return null != indexResultsIterator && indexResultsIterator.hasNext();
-        }
-
-        public boolean hasGremlinResults() {
-            return null != gremlinResultIterator && gremlinResultIterator.hasNext();
-        }
-
-
-        public boolean isTagProcessingComplete() {
-            return CollectionUtils.isEmpty(tagSearchAttributes) ||
-                    CollectionUtils.isEqualCollection(tagSearchAttributes, tagAttrProcessedBySolr);
-        }
-
-        public boolean isEntityProcessingComplete() {
-            return CollectionUtils.isEmpty(entitySearchAttributes) ||
-                    CollectionUtils.isEqualCollection(entitySearchAttributes, entityAttrProcessedBySolr);
-        }
-
-        public boolean isProcessingComplete() {
-            return isTagProcessingComplete() && isEntityProcessingComplete();
-        }
-
-        public void incrementOffset(int increment) {
-            currentOffset += increment;
-        }
-
-        public void incrementSearchRound() {
-            iterationCount ++;
-            incrementOffset(searchParameters.getLimit());
-        }
-
-        public int getCurrentOffset() {
-            return currentOffset;
-        }
-
-        public boolean addTagSearchAttribute(String attribute) {
-            return tagSearchAttributes.add(attribute);
-        }
-
-        public boolean addProcessedTagAttribute(String attribute) {
-            return tagAttrProcessedBySolr.add(attribute);
-        }
-
-        public boolean addEntitySearchAttribute(String attribute) {
-            return tagSearchAttributes.add(attribute);
-        }
-
-        public boolean addProcessedEntityAttribute(String attribute) {
-            return entityAttrProcessedBySolr.add(attribute);
-        }
-
-        public void cacheGraphQuery(String name, AtlasGraphQuery graphQuery) {
-            cachedGraphQueries.put(name, graphQuery);
-        }
-
-        public void cacheIndexQuery(String name, AtlasIndexQuery indexQuery) {
-            cachedIndexQueries.put(name, indexQuery);
-        }
-
-        public AtlasIndexQuery getIndexQuery(String name){
-            return cachedIndexQueries.get(name);
-        }
-
-        public AtlasGraphQuery getGraphQuery(String name) {
-            return cachedGraphQueries.get(name);
-        }
-
-        public IndexResultType getIndexResultType() {
-            return indexResultType;
-        }
-
-        public void setIndexResultType(IndexResultType indexResultType) {
-            this.indexResultType = indexResultType;
-        }
-
-        public int getMaxLimit() {
-            return maxLimit;
-        }
-
-        @Override
-        public String toString() {
-            return new ToStringBuilder(this)
-                    .append("iterationCount", iterationCount)
-                    .append("forceTerminate", forceTerminate)
-                    .append("currentOffset", currentOffset)
-                    .append("maxLimit", maxLimit)
-                    .append("searchParameters", searchParameters)
-                    .append("tagSearchAttributes", tagSearchAttributes)
-                    .append("entitySearchAttributes", entitySearchAttributes)
-                    .append("tagAttrProcessedBySolr", tagAttrProcessedBySolr)
-                    .append("entityAttrProcessedBySolr", entityAttrProcessedBySolr)
-                    .append("indexResultType", indexResultType)
-                    .append("cachedIndexQueries", cachedIndexQueries)
-                    .append("cachedGraphQueries", cachedGraphQueries)
-                    .toString();
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
new file mode 100644
index 0000000..1a2d997
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -0,0 +1,381 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.regex.Pattern;
+
+public abstract class SearchProcessor {
+    private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class);
+
+    public static final Pattern STRAY_AND_PATTERN                       = Pattern.compile("(AND\\s+)+\\)");
+    public static final Pattern STRAY_OR_PATTERN                        = Pattern.compile("(OR\\s+)+\\)");
+    public static final Pattern STRAY_ELIPSIS_PATTERN                   = Pattern.compile("(\\(\\s*)\\)");
+    public static final int     MAX_RESULT_SIZE                         = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
+    public static final int     MAX_ENTITY_TYPES_IN_INDEX_QUERY         = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10);
+    public static final int     MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10);
+    public static final String  AND_STR         = " AND ";
+    public static final String  EMPTY_STRING    = "";
+    public static final String  SPACE_STRING    = " ";
+    public static final String  BRACE_OPEN_STR  = "( ";
+    public static final String  BRACE_CLOSE_STR = " )";
+
+    private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
+
+    static
+    {
+        OPERATOR_MAP.put(SearchParameters.Operator.LT,"v.\"%s\": [* TO %s}");
+        OPERATOR_MAP.put(SearchParameters.Operator.GT,"v.\"%s\": {%s TO *]");
+        OPERATOR_MAP.put(SearchParameters.Operator.LTE,"v.\"%s\": [* TO %s]");
+        OPERATOR_MAP.put(SearchParameters.Operator.GTE,"v.\"%s\": [%s TO *]");
+        OPERATOR_MAP.put(SearchParameters.Operator.EQ,"v.\"%s\": %s");
+        OPERATOR_MAP.put(SearchParameters.Operator.NEQ,"v.\"%s\": (NOT %s)");
+        OPERATOR_MAP.put(SearchParameters.Operator.IN, "v.\"%s\": (%s)");
+        OPERATOR_MAP.put(SearchParameters.Operator.LIKE, "v.\"%s\": (%s)");
+        OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, "v.\"%s\": (%s*)");
+        OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, "v.\"%s\": (*%s)");
+        OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, "v.\"%s\": (*%s*)");
+    }
+
+    protected final SearchContext   context;
+    protected       SearchProcessor nextProcessor;
+
+
+    protected SearchProcessor(SearchContext context) {
+        this.context = context;
+    }
+
+    public void addProcessor(SearchProcessor processor) {
+        if (nextProcessor == null) {
+            nextProcessor = processor;
+        } else {
+            nextProcessor.addProcessor(processor);
+        }
+    }
+
+    public abstract List<AtlasVertex> execute();
+
+    public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
+        return nextProcessor == null ? entityVertices : nextProcessor.filter(entityVertices);
+    }
+
+
+    protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> solrFiltered, Set<String> gremlinFiltered, Set<String> allAttributes) {
+        if (structType == null || filterCriteria == null) {
+            return;
+        }
+
+        Condition            filterCondition = filterCriteria.getCondition();
+        List<FilterCriteria> criterion       = filterCriteria.getCriterion();
+
+        if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
+            for (SearchParameters.FilterCriteria criteria : criterion) {
+                processSearchAttributes(structType, criteria, solrFiltered, gremlinFiltered, allAttributes);
+            }
+        } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
+            try {
+                String      attributeName = filterCriteria.getAttributeName();
+                String      qualifiedName = structType.getQualifiedAttributeName(attributeName);
+                Set<String> indexedKeys   = context.getIndexedKeys();
+
+                if (indexedKeys != null && indexedKeys.contains(qualifiedName)) {
+                    solrFiltered.add(attributeName);
+                } else {
+                    LOG.warn("search includes non-indexed attribute '{}'; might cause poor performance", qualifiedName);
+
+                    gremlinFiltered.add(attributeName);
+                }
+
+                allAttributes.add(attributeName);
+            } catch (AtlasBaseException e) {
+                LOG.warn(e.getMessage());
+            }
+        }
+    }
+
+    //
+    // If filterCriteria contains any non-indexed attribute inside OR condition:
+    //    Solr+Grelin can't be used. Need to use only Gremlin filter for all attributes. Examples:
+    //    (OR idx-att1=x non-idx-attr=z)
+    //    (AND idx-att1=x (OR idx-attr2=y non-idx-attr=z))
+    // Else
+    //    Solr can be used for indexed-attribute filtering and Gremlin for non-indexed attributes. Examples:
+    //      (AND idx-att1=x idx-attr2=y non-idx-attr=z)
+    //      (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z)
+    //      (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc))
+    //
+    protected boolean canApplySolrFilter(AtlasStructType structType, FilterCriteria filterCriteria, boolean insideOrCondition) {
+        if (filterCriteria == null) {
+            return true;
+        }
+
+        boolean              ret             = true;
+        Condition            filterCondition = filterCriteria.getCondition();
+        List<FilterCriteria> criterion       = filterCriteria.getCriterion();
+        Set<String>          indexedKeys     = context.getIndexedKeys();
+
+
+        if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
+            insideOrCondition = insideOrCondition || filterCondition == Condition.OR;
+
+            // If we have nested criterion let's find any nested ORs with non-indexed attr
+            for (FilterCriteria criteria : criterion) {
+                ret = canApplySolrFilter(structType, criteria, insideOrCondition);
+
+                if (!ret) {
+                    break;
+                }
+            }
+        } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
+            try {
+                String qualifiedName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
+
+                if (insideOrCondition && (indexedKeys == null || !indexedKeys.contains(qualifiedName))) {
+                    ret = false;
+                }
+            } catch (AtlasBaseException e) {
+                LOG.warn(e.getMessage());
+            }
+        }
+
+        return ret;
+    }
+
+    protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
+        String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
+
+        solrQuery.append("v.\"__typeName\": (")
+                .append(typeAndSubtypesString)
+                .append(")");
+    }
+
+    protected void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> solrAttributes) {
+        if (filterCriteria != null) {
+            LOG.debug("Processing Filters");
+
+            String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes);
+
+            if (StringUtils.isNotEmpty(filterQuery)) {
+                solrQuery.append(AND_STR).append(filterQuery);
+            }
+        }
+
+        if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
+            solrQuery.append(AND_STR).append("v.\"__state\":").append("ACTIVE");
+        }
+    }
+
+    private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes) {
+        return toSolrQuery(type, criteria, solrAttributes, new StringBuilder());
+    }
+
+    private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, StringBuilder sb) {
+        if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
+            StringBuilder nestedExpression = new StringBuilder();
+
+            for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+                String nestedQuery = toSolrQuery(type, filterCriteria, solrAttributes);
+
+                if (StringUtils.isNotEmpty(nestedQuery)) {
+                    if (nestedExpression.length() > 0) {
+                        nestedExpression.append(SPACE_STRING).append(criteria.getCondition()).append(SPACE_STRING);
+                    }
+
+                    nestedExpression.append(nestedQuery);
+                }
+            }
+
+            return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression.toString()).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING;
+        } else if (solrAttributes.contains(criteria.getAttributeName())){
+            return toSolrExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
+        } else {
+            return EMPTY_STRING;
+        }
+    }
+
+    private String toSolrExpression(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) {
+        String ret = EMPTY_STRING;
+
+        try {
+            String qualifiedName = type.getQualifiedAttributeName(attrName);
+
+            if (OPERATOR_MAP.get(op) != null) {
+                ret = String.format(OPERATOR_MAP.get(op), qualifiedName, attrVal);
+            }
+        } catch (AtlasBaseException ex) {
+            LOG.warn(ex.getMessage());
+        }
+
+        return ret;
+    }
+
+    protected AtlasGraphQuery toGremlinFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> gremlinAttributes, AtlasGraphQuery query) {
+        if (criteria != null) {
+            if (criteria.getCondition() != null) {
+                if (criteria.getCondition() == Condition.AND) {
+                    for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+                        AtlasGraphQuery nestedQuery = toGremlinFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
+
+                        query.addConditionsFrom(nestedQuery);
+                    }
+                } else {
+                    List<AtlasGraphQuery> orConditions = new LinkedList<>();
+
+                    for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+                        AtlasGraphQuery nestedQuery = toGremlinFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
+
+                        orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
+                    }
+
+                    if (!orConditions.isEmpty()) {
+                        query.or(orConditions);
+                    }
+                }
+            } else if (gremlinAttributes.contains(criteria.getAttributeName())) {
+                String                    attrName  = criteria.getAttributeName();
+                String                    attrValue = criteria.getAttributeValue();
+                SearchParameters.Operator operator  = criteria.getOperator();
+
+                try {
+                    final String qualifiedName = type.getQualifiedAttributeName(attrName);
+
+                    switch (operator) {
+                        case LT:
+                            query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
+                            break;
+                        case LTE:
+                            query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
+                            break;
+                        case GT:
+                            query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
+                            break;
+                        case GTE:
+                            query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
+                            break;
+                        case EQ:
+                            query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
+                            break;
+                        case NEQ:
+                            query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
+                            break;
+                        case LIKE:
+                            // TODO: Maybe we need to validate pattern
+                            query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getLikeRegex(attrValue));
+                            break;
+                        case CONTAINS:
+                            query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
+                            break;
+                        case STARTS_WITH:
+                            query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
+                            break;
+                        case ENDS_WITH:
+                            query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
+                            break;
+                        case IN:
+                            LOG.warn("{}: unsupported operator. Ignored", operator);
+                            break;
+                    }
+                } catch (AtlasBaseException e) {
+                    LOG.error("toGremlinFilterQuery(): failed for attrName=" + attrName + "; operator=" + operator + "; attrValue=" + attrValue, e);
+                }
+            }
+        }
+
+        return query;
+    }
+
+    private String getContainsRegex(String attributeValue) {
+        return ".*" + attributeValue + ".*";
+    }
+
+    private String getSuffixRegex(String attributeValue) {
+        return ".*" + attributeValue;
+    }
+
+    private String getLikeRegex(String attributeValue) { return ".*" + attributeValue + ".*"; }
+
+    protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult) {
+        List<AtlasVertex> ret = new ArrayList<>();
+
+        if (idxQueryResult != null) {
+            while (idxQueryResult.hasNext()) {
+                AtlasVertex vertex = idxQueryResult.next().getVertex();
+
+                ret.add(vertex);
+            }
+        }
+
+        return ret;
+    }
+
+    protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> vertices) {
+        List<AtlasVertex> ret = new ArrayList<>();
+
+        if (vertices != null) {
+            while (vertices.hasNext()) {
+                AtlasVertex vertex = vertices.next();
+
+                ret.add(vertex);
+            }
+        }
+
+        return ret;
+    }
+
+    protected Set<String> getGuids(List<AtlasVertex> vertices) {
+        Set<String> ret = new HashSet<>();
+
+        if (vertices != null) {
+            for(AtlasVertex vertex : vertices) {
+                String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
+
+                if (StringUtils.isNotEmpty(guid)) {
+                    ret.add(guid);
+                }
+            }
+        }
+
+        return ret;
+    }
+
+    private static int getApplicationProperty(String propertyName, int defaultValue) {
+        try {
+            return ApplicationProperties.get().getInt(propertyName, defaultValue);
+        } catch (AtlasException excp) {
+            // ignore
+        }
+
+        return defaultValue;
+    }
+}

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java b/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java
deleted file mode 100644
index 6a5dd5a..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.discovery.SearchPipeline.IndexResultType;
-import org.apache.atlas.discovery.SearchPipeline.PipelineContext;
-import org.apache.atlas.discovery.SearchPipeline.PipelineStep;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.SearchParameters.Operator;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
-import org.apache.atlas.type.*;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import static org.apache.atlas.model.discovery.SearchParameters.*;
-
-@Component
-public class SolrStep implements PipelineStep {
-    private static final Logger LOG = LoggerFactory.getLogger(SolrStep.class);
-
-    private static final Pattern STRAY_AND_PATTERN     = Pattern.compile("(AND\\s+)+\\)");
-    private static final Pattern STRAY_OR_PATTERN      = Pattern.compile("(OR\\s+)+\\)");
-    private static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
-    private static final String  AND_STR         = " AND ";
-    private static final String  EMPTY_STRING    = "";
-    private static final String  SPACE_STRING    = " ";
-    private static final String  BRACE_OPEN_STR  = "( ";
-    private static final String  BRACE_CLOSE_STR = " )";
-
-    private static final Map<Operator, String> operatorMap = new HashMap<>();
-
-    static
-    {
-        operatorMap.put(Operator.LT,"v.\"%s\": [* TO %s}");
-        operatorMap.put(Operator.GT,"v.\"%s\": {%s TO *]");
-        operatorMap.put(Operator.LTE,"v.\"%s\": [* TO %s]");
-        operatorMap.put(Operator.GTE,"v.\"%s\": [%s TO *]");
-        operatorMap.put(Operator.EQ,"v.\"%s\": %s");
-        operatorMap.put(Operator.NEQ,"v.\"%s\": (NOT %s)");
-        operatorMap.put(Operator.IN, "v.\"%s\": (%s)");
-        operatorMap.put(Operator.LIKE, "v.\"%s\": (%s)");
-        operatorMap.put(Operator.STARTS_WITH, "v.\"%s\": (%s*)");
-        operatorMap.put(Operator.ENDS_WITH, "v.\"%s\": (*%s)");
-        operatorMap.put(Operator.CONTAINS, "v.\"%s\": (*%s*)");
-    }
-
-    private final AtlasGraph graph;
-
-    @Inject
-    public SolrStep(AtlasGraph graph) {
-        this.graph = graph;
-    }
-
-    @Override
-    public void execute(PipelineContext context) throws AtlasBaseException {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("==> SolrStep.execute({})", context);
-        }
-
-        if (context == null) {
-            throw new AtlasBaseException("Can't start search without any context");
-        }
-
-        SearchParameters searchParameters = context.getSearchParameters();
-
-        final Iterator<AtlasIndexQuery.Result> result;
-
-        if (StringUtils.isNotEmpty(searchParameters.getQuery())) {
-            result = executeAgainstFulltextIndex(context);
-        } else {
-            result = executeAgainstVertexIndex(context);
-        }
-
-        context.setIndexResultsIterator(result);
-
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("<== SolrStep.execute({})", context);
-        }
-    }
-
-    private Iterator<AtlasIndexQuery.Result> executeAgainstFulltextIndex(PipelineContext context) {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("==> SolrStep.executeAgainstFulltextIndex()");
-        }
-
-        final Iterator<AtlasIndexQuery.Result> ret;
-
-        AtlasIndexQuery query = context.getIndexQuery("FULLTEXT");
-
-        if (query == null) {
-            // Compute only once
-            SearchParameters searchParameters = context.getSearchParameters();
-            String           indexQuery       = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, searchParameters.getQuery());
-
-            query = graph.indexQuery(Constants.FULLTEXT_INDEX, indexQuery);
-
-            context.cacheIndexQuery("FULLTEXT", query);
-        }
-
-        context.setIndexResultType(IndexResultType.TEXT);
-
-        ret = query.vertices(context.getCurrentOffset(), context.getMaxLimit());
-
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("<== SolrStep.executeAgainstFulltextIndex()");
-        }
-
-        return ret;
-    }
-
-    private Iterator<AtlasIndexQuery.Result> executeAgainstVertexIndex(PipelineContext context) {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("==> SolrStep.executeAgainstVertexIndex()");
-        }
-
-        final Iterator<AtlasIndexQuery.Result> ret;
-
-        SearchParameters searchParameters = context.getSearchParameters();
-        AtlasIndexQuery  query            = context.getIndexQuery("VERTEX_INDEX");
-
-        if (query == null) {
-            StringBuilder solrQuery = new StringBuilder();
-
-            // If tag is specified then let's start processing using tag and it's attributes, entity filters will
-            // be pushed to Gremlin
-            if (context.getClassificationType() != null) {
-                context.setIndexResultType(IndexResultType.TAG);
-
-                constructTypeTestQuery(solrQuery, context.getClassificationType().getTypeAndAllSubTypes());
-                constructFilterQuery(solrQuery, context.getClassificationType(), searchParameters.getTagFilters(), context);
-            } else if (context.getEntityType() != null) {
-                context.setIndexResultType(IndexResultType.ENTITY);
-
-                constructTypeTestQuery(solrQuery, context.getEntityType().getTypeAndAllSubTypes());
-                constructFilterQuery(solrQuery, context.getEntityType(), searchParameters.getEntityFilters(), context);
-
-                // Set the status flag
-                if (searchParameters.getExcludeDeletedEntities()) {
-                    if (solrQuery.length() > 0) {
-                        solrQuery.append(" AND ");
-                    }
-
-                    solrQuery.append("v.\"__state\":").append("ACTIVE");
-                }
-            }
-
-            // No query was formed, doesn't make sense to do anything beyond this point
-            if (solrQuery.length() > 0) {
-                String validSolrQuery = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
-                validSolrQuery = STRAY_OR_PATTERN.matcher(validSolrQuery).replaceAll(")");
-                validSolrQuery = STRAY_ELIPSIS_PATTERN.matcher(validSolrQuery).replaceAll(EMPTY_STRING);
-
-                query = graph.indexQuery(Constants.VERTEX_INDEX, validSolrQuery);
-                context.cacheIndexQuery("VERTEX_INDEX", query);
-            }
-        }
-
-        // Execute solr query and return the index results in the context
-        if (query != null) {
-            ret = query.vertices(context.getCurrentOffset(), context.getMaxLimit());
-        } else {
-            ret = null;
-        }
-
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("<== SolrStep.executeAgainstVertexIndex()");
-        }
-
-        return ret;
-    }
-
-    private void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
-        String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
-
-        solrQuery.append("v.\"__typeName\": (")
-                .append(typeAndSubtypesString)
-                .append(")");
-    }
-
-    private void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, PipelineContext context) {
-        if (filterCriteria != null) {
-            LOG.debug("Processing Filters");
-
-            String filterQuery = toSolrQuery(type, filterCriteria, context);
-
-            if (StringUtils.isNotEmpty(filterQuery)) {
-                solrQuery.append(AND_STR).append(filterQuery);
-            }
-        }
-    }
-
-    private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, PipelineContext context) {
-        return toSolrQuery(type, criteria, context, new StringBuilder());
-    }
-
-    private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, PipelineContext context, StringBuilder sb) {
-        if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
-            StringBuilder nestedExpression = new StringBuilder();
-
-            for (FilterCriteria filterCriteria : criteria.getCriterion()) {
-                String nestedQuery = toSolrQuery(type, filterCriteria, context);
-
-                if (StringUtils.isNotEmpty(nestedQuery)) {
-                    if (nestedExpression.length() > 0) {
-                        nestedExpression.append(SPACE_STRING).append(criteria.getCondition()).append(SPACE_STRING);
-                    }
-
-                    nestedExpression.append(nestedQuery);
-                }
-            }
-
-            return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression.toString()).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING;
-        } else {
-            return toSolrExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue(), context);
-        }
-    }
-
-    private String toSolrExpression(AtlasStructType type, String attrName, Operator op, String attrVal, PipelineContext context) {
-        String ret = EMPTY_STRING;
-
-        try {
-            String    indexKey      = type.getQualifiedAttributeName(attrName);
-            AtlasType attributeType = type.getAttributeType(attrName);
-
-            switch (context.getIndexResultType()) {
-                case TAG:
-                    context.addTagSearchAttribute(indexKey);
-                    break;
-
-                case ENTITY:
-                    context.addEntitySearchAttribute(indexKey);
-                    break;
-
-                default:
-                    // Do nothing
-            }
-
-            if (attributeType != null && AtlasTypeUtil.isBuiltInType(attributeType.getTypeName()) && context.getIndexedKeys().contains(indexKey)) {
-                if (operatorMap.get(op) != null) {
-                    // If there's a chance of multi-value then we need some additional processing here
-                    switch (context.getIndexResultType()) {
-                        case TAG:
-                            context.addProcessedTagAttribute(indexKey);
-                            break;
-
-                        case ENTITY:
-                            context.addProcessedEntityAttribute(indexKey);
-                            break;
-                    }
-
-                    ret = String.format(operatorMap.get(op), indexKey, attrVal);
-                }
-            }
-        } catch (AtlasBaseException ex) {
-            LOG.warn(ex.getMessage());
-        }
-
-        return ret;
-    }
-}

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 4ac00a3..f4257be 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -128,6 +128,10 @@ public final class EntityGraphRetriever {
         return ret;
     }
 
+    public AtlasEntityHeader toAtlasEntityHeader(String guid) throws AtlasBaseException {
+        return toAtlasEntityHeader(getEntityVertex(guid));
+    }
+
     public AtlasEntityHeader toAtlasEntityHeader(AtlasVertex entityVertex) throws AtlasBaseException {
         return toAtlasEntityHeader(entityVertex, Collections.<String>emptySet());
     }
@@ -233,14 +237,15 @@ public final class EntityGraphRetriever {
 
             if (CollectionUtils.isNotEmpty(attributes)) {
                 for (String attrName : attributes) {
+                    String nonQualifiedAttrName = toNonQualifiedName(attrName);
                     if (ret.hasAttribute(attrName)) {
                         continue;
                     }
 
-                    Object attrValue = getVertexAttribute(entityVertex, entityType.getAttribute(attrName));
+                    Object attrValue = getVertexAttribute(entityVertex, entityType.getAttribute(nonQualifiedAttrName));
 
                     if (attrValue != null) {
-                        ret.setAttribute(attrName, attrValue);
+                        ret.setAttribute(nonQualifiedAttrName, attrValue);
                     }
                 }
             }
@@ -249,6 +254,17 @@ public final class EntityGraphRetriever {
         return ret;
     }
 
+    private String toNonQualifiedName(String attrName) {
+        String ret;
+        if (attrName.contains(".")) {
+            String[] attributeParts = attrName.split("\\.");
+            ret = attributeParts[attributeParts.length - 1];
+        } else {
+            ret = attrName;
+        }
+        return ret;
+    }
+
     private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("Mapping system attributes for type {}", entity.getTypeName());

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/util/SearchTracker.java b/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
index 15a8c20..e58e855 100644
--- a/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
+++ b/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
@@ -18,7 +18,7 @@
 package org.apache.atlas.util;
 
 import org.apache.atlas.annotation.AtlasService;
-import org.apache.atlas.discovery.SearchPipeline.PipelineContext;
+import org.apache.atlas.discovery.SearchContext;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -26,13 +26,13 @@ import java.util.Set;
 
 @AtlasService
 public class SearchTracker {
-    private Map<String, PipelineContext> activeSearches = new HashMap<>();
+    private Map<String, SearchContext> activeSearches = new HashMap<>();
 
     /**
      *
      * @param context
      */
-    public String add(PipelineContext context) {
+    public String add(SearchContext context) {
         String searchId = Thread.currentThread().getName();
 
         activeSearches.put(searchId, context);
@@ -45,13 +45,13 @@ public class SearchTracker {
      * @param searchId
      * @return
      */
-    public PipelineContext terminate(String searchId) {
-        PipelineContext ret = null;
+    public SearchContext terminate(String searchId) {
+        SearchContext ret = null;
 
         if (activeSearches.containsKey(searchId)) {
-            PipelineContext pipelineToTerminate = activeSearches.remove(searchId);
+            SearchContext pipelineToTerminate = activeSearches.remove(searchId);
 
-            pipelineToTerminate.setForceTerminate(true);
+            pipelineToTerminate.terminateSearch(true);
 
             ret = pipelineToTerminate;
         }
@@ -59,7 +59,7 @@ public class SearchTracker {
         return ret;
     }
 
-    public PipelineContext remove(String id) {
+    public SearchContext remove(String id) {
         return activeSearches.remove(id);
     }
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/test/java/org/apache/atlas/TestModules.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java
index d0da030..4bc1487 100644
--- a/repository/src/test/java/org/apache/atlas/TestModules.java
+++ b/repository/src/test/java/org/apache/atlas/TestModules.java
@@ -143,10 +143,7 @@ public class TestModules {
             typeDefChangeListenerMultibinder.addBinding().to(DefaultMetadataService.class);
             typeDefChangeListenerMultibinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
 
-            bind(SearchPipeline.class).asEagerSingleton();
             bind(SearchTracker.class).asEagerSingleton();
-            bind(SolrStep.class).asEagerSingleton();
-            bind(GremlinStep.class).asEagerSingleton();
 
             bind(AtlasEntityStore.class).to(AtlasEntityStoreV1.class);
             bind(AtlasRelationshipStore.class).to(AtlasRelationshipStoreV1.class);

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 1a9f57a..2a5fc90 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -25,7 +25,7 @@ import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.authorize.AtlasActionTypes;
 import org.apache.atlas.authorize.AtlasResourceTypes;
 import org.apache.atlas.authorize.simple.AtlasAuthorizationUtils;
-import org.apache.atlas.discovery.SearchPipeline;
+import org.apache.atlas.discovery.SearchContext;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasExportRequest;
 import org.apache.atlas.model.impexp.AtlasExportResult;
@@ -434,7 +434,7 @@ public class AdminResource {
     @Path("activeSearches/{id}")
     @Produces(Servlets.JSON_MEDIA_TYPE)
     public boolean terminateActiveSearch(@PathParam("id") String searchId) {
-        SearchPipeline.PipelineContext terminate = activeSearches.terminate(searchId);
+        SearchContext terminate = activeSearches.terminate(searchId);
         return null != terminate;
     }
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index dde300e..efab72a 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -249,7 +249,7 @@ public class DiscoveryREST {
                 throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "TagFilters specified without tag name");
             }
 
-            return atlasDiscoveryService.searchUsingBasicQuery(parameters);
+            return atlasDiscoveryService.searchWithParameters(parameters);
         } finally {
             AtlasPerfTracer.log(perf);
         }